You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/11/24 10:43:12 UTC
[01/49] hadoop git commit: YARN-7218. Decouple YARN Services REST API
namespace from RM. (Contributed by Eric Yang) [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5881 d0af8b829 -> 85fc1b517 (forced update)
YARN-7218. Decouple YARN Services REST API namespace from RM. (Contributed by Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0940e4f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0940e4f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0940e4f6
Branch: refs/heads/YARN-5881
Commit: 0940e4f692441f16e742666ac925f71a083eab27
Parents: 5f0b238
Author: Eric Yang <ey...@apache.org>
Authored: Fri Nov 17 12:28:12 2017 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Nov 17 12:28:12 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/service/TestApiServer.java | 4 +-
.../yarn/service/api/records/Component.java | 51 +++++++++++++-------
.../service/api/records/ReadinessCheck.java | 14 +++++-
.../yarn/service/api/records/Service.java | 15 +++---
.../yarn/service/conf/RestApiConstants.java | 2 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 10 +++-
.../server/resourcemanager/ResourceManager.java | 14 +++++-
.../server/resourcemanager/webapp/RMWebApp.java | 15 ------
8 files changed, 77 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
index 2b22474..896b2f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -62,8 +62,8 @@ public class TestApiServer {
this.apiServer.getClass().isAnnotationPresent(Path.class));
final Path path = this.apiServer.getClass()
.getAnnotation(Path.class);
- assertEquals("The path has /ws/v1 annotation", path.value(),
- "/ws/v1");
+ assertEquals("The path has /v1 annotation", path.value(),
+ "/v1");
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
index fe9c043..ce0e0cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -26,6 +26,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@@ -46,22 +48,53 @@ import org.apache.hadoop.classification.InterfaceStability;
@ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Component implements Serializable {
private static final long serialVersionUID = -8430058381509087805L;
+ @JsonProperty("name")
private String name = null;
+
+ @JsonProperty("dependencies")
private List<String> dependencies = new ArrayList<String>();
+
+ @JsonProperty("readiness_check")
+ @XmlElement(name = "readiness_check")
private ReadinessCheck readinessCheck = null;
+
+ @JsonProperty("artifact")
private Artifact artifact = null;
+
+ @JsonProperty("launch_command")
+ @XmlElement(name = "launch_command")
private String launchCommand = null;
+
+ @JsonProperty("resource")
private Resource resource = null;
+
+ @JsonProperty("number_of_containers")
+ @XmlElement(name = "number_of_containers")
private Long numberOfContainers = null;
+
+ @JsonProperty("run_privileged_container")
+ @XmlElement(name = "run_privileged_container")
private Boolean runPrivilegedContainer = false;
+
+ @JsonProperty("placement_policy")
+ @XmlElement(name = "placement_policy")
private PlacementPolicy placementPolicy = null;
+
+ @JsonProperty("state")
private ComponentState state = ComponentState.FLEXING;
+
+ @JsonProperty("configuration")
private Configuration configuration = new Configuration();
+
+ @JsonProperty("quicklinks")
private List<String> quicklinks = new ArrayList<String>();
+
+ @JsonProperty("containers")
private List<Container> containers =
Collections.synchronizedList(new ArrayList<Container>());
@@ -74,7 +107,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).")
- @JsonProperty("name")
public String getName() {
return name;
}
@@ -95,7 +127,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.")
- @JsonProperty("dependencies")
public List<String> getDependencies() {
return dependencies;
}
@@ -113,12 +144,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Readiness check for this component.")
- @JsonProperty("readiness_check")
public ReadinessCheck getReadinessCheck() {
return readinessCheck;
}
- @XmlElement(name = "readiness_check")
public void setReadinessCheck(ReadinessCheck readinessCheck) {
this.readinessCheck = readinessCheck;
}
@@ -133,7 +162,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.")
- @JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
}
@@ -153,12 +181,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
- @JsonProperty("launch_command")
public String getLaunchCommand() {
return launchCommand;
}
- @XmlElement(name = "launch_command")
public void setLaunchCommand(String launchCommand) {
this.launchCommand = launchCommand;
}
@@ -173,7 +199,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.")
- @JsonProperty("resource")
public Resource getResource() {
return resource;
}
@@ -192,18 +217,15 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.")
- @JsonProperty("number_of_containers")
public Long getNumberOfContainers() {
return numberOfContainers;
}
- @XmlElement(name = "number_of_containers")
public void setNumberOfContainers(Long numberOfContainers) {
this.numberOfContainers = numberOfContainers;
}
@ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
- @JsonProperty("containers")
public List<Container> getContainers() {
return containers;
}
@@ -237,12 +259,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
- @JsonProperty("run_privileged_container")
public Boolean getRunPrivilegedContainer() {
return runPrivilegedContainer;
}
- @XmlElement(name = "run_privileged_container")
public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
this.runPrivilegedContainer = runPrivilegedContainer;
}
@@ -259,12 +279,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.")
- @JsonProperty("placement_policy")
public PlacementPolicy getPlacementPolicy() {
return placementPolicy;
}
- @XmlElement(name = "placement_policy")
public void setPlacementPolicy(PlacementPolicy placementPolicy) {
this.placementPolicy = placementPolicy;
}
@@ -278,7 +296,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Config properties for this component.")
- @JsonProperty("configuration")
public Configuration getConfiguration() {
return configuration;
}
@@ -297,7 +314,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.")
- @JsonProperty("quicklinks")
public List<String> getQuicklinks() {
return quicklinks;
}
@@ -312,7 +328,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "State of the component.")
- @JsonProperty("state")
public ComponentState getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index 0a3713c..2bcf68b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,7 +25,11 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -43,6 +47,8 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
@@ -68,8 +74,14 @@ public class ReadinessCheck implements Serializable {
}
}
+ @JsonProperty("type")
+ @XmlElement(name = "type")
private TypeEnum type = null;
+ @JsonProperty("properties")
+ @XmlElement(name = "properties")
private Map<String, String> properties = new HashMap<String, String>();
+ @JsonProperty("artifact")
+ @XmlElement(name = "artifact")
private Artifact artifact = null;
/**
@@ -82,7 +94,6 @@ public class ReadinessCheck implements Serializable {
}
@ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
- @JsonProperty("type")
public TypeEnum getType() {
return type;
}
@@ -129,7 +140,6 @@ public class ReadinessCheck implements Serializable {
}
@ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.")
- @JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
index 77a2610..8045822 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -26,6 +26,8 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
@@ -43,6 +45,7 @@ import java.util.Objects;
@ApiModel(description = "An Service resource has the following attributes.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
"lifetime", "containers" })
@@ -53,9 +56,15 @@ public class Service extends BaseResource {
private String id = null;
private Artifact artifact = null;
private Resource resource = null;
+ @JsonProperty("launch_time")
+ @XmlElement(name = "launch_time")
private Date launchTime = null;
+ @JsonProperty("number_of_running_containers")
+ @XmlElement(name = "number_of_running_containers")
private Long numberOfRunningContainers = null;
private Long lifetime = null;
+ @JsonProperty("placement_policy")
+ @XmlElement(name = "placement_policy")
private PlacementPolicy placementPolicy = null;
private List<Component> components = new ArrayList<>();
private Configuration configuration = new Configuration();
@@ -148,12 +157,10 @@ public class Service extends BaseResource {
}
@ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.")
- @JsonProperty("launch_time")
public Date getLaunchTime() {
return launchTime == null ? null : (Date) launchTime.clone();
}
- @XmlElement(name = "launch_time")
public void setLaunchTime(Date launchTime) {
this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
}
@@ -171,12 +178,10 @@ public class Service extends BaseResource {
}
@ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
- @JsonProperty("number_of_running_containers")
public Long getNumberOfRunningContainers() {
return numberOfRunningContainers;
}
- @XmlElement(name = "number_of_running_containers")
public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
this.numberOfRunningContainers = numberOfRunningContainers;
}
@@ -215,12 +220,10 @@ public class Service extends BaseResource {
}
@ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
- @JsonProperty("placement_policy")
public PlacementPolicy getPlacementPolicy() {
return placementPolicy;
}
- @XmlElement(name = "placement_policy")
public void setPlacementPolicy(PlacementPolicy placementPolicy) {
this.placementPolicy = placementPolicy;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
index 35e1980..243fc52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.conf;
public interface RestApiConstants {
// Rest endpoints
- String CONTEXT_ROOT = "/ws/v1";
+ String CONTEXT_ROOT = "/v1";
String VERSION = "/services/version";
String SERVICE_ROOT_PATH = "/services";
String SERVICE_PATH = "/services/{service_name}";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 4f1cacf..d3ad53e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -394,16 +394,22 @@ public class WebApps {
}
public WebApp start(WebApp webapp) {
- return start(webapp, null);
+ return start(webapp, null, null);
}
- public WebApp start(WebApp webapp, WebAppContext ui2Context) {
+ public WebApp start(WebApp webapp, WebAppContext ui2Context,
+ Map<String, String> services) {
WebApp webApp = build(webapp);
HttpServer2 httpServer = webApp.httpServer();
if (ui2Context != null) {
addFiltersForNewContext(ui2Context);
httpServer.addHandlerAtFront(ui2Context);
}
+ if (services!=null) {
+ String packageName = services.get("PackageName");
+ String pathSpec = services.get("PathSpec");
+ httpServer.addJerseyResourcePackage(packageName, pathSpec);
+ }
try {
httpServer.start();
LOG.info("Web app " + name + " started at "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 727bc52..6f8a0a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -126,7 +126,9 @@ import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
import java.security.SecureRandom;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -1062,7 +1064,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected void startWepApp() {
-
+ Map<String, String> serviceConfig = null;
Configuration conf = getConfig();
RMWebAppUtil.setupSecurityAndFilters(conf,
@@ -1128,7 +1130,15 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
- webApp = builder.start(new RMWebApp(this), uiWebAppContext);
+ if (getConfig().getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
+ false)) {
+ serviceConfig = new HashMap<String, String>();
+ String apiPackages = "org.apache.hadoop.yarn.service.webapp;" +
+ "org.apache.hadoop.yarn.webapp";
+ serviceConfig.put("PackageName", apiPackages);
+ serviceConfig.put("PathSpec", "/app/*");
+ }
+ webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig);
}
private String getWebAppsPath(String appName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index bee9354..b8d4390 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -44,8 +44,6 @@ public class RMWebApp extends WebApp implements YarnWebParams {
LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm;
private boolean standby = false;
- private final static String APISERVER =
- "org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) {
this.rm = rm;
@@ -59,19 +57,6 @@ public class RMWebApp extends WebApp implements YarnWebParams {
bind(RMWebApp.class).toInstance(this);
if (rm != null) {
- boolean enableServiceApi = rm.getConfig()
- .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
- if (enableServiceApi) {
- try {
- // Use reflection here to load ApiServer class,
- // this is done to avoid creating cyclic dependency
- // between maven projects.
- Class<?> apiServer = Class.forName(APISERVER);
- bind(apiServer);
- } catch (ClassNotFoundException e) {
- LOG.warn("ApiServer REST API is not activated.");
- }
- }
bind(ResourceManager.class).toInstance(rm);
}
route("/", RmController.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
HADOOP-13786 Add S3A committer for zero-rename commits to S3 endpoints.
Contributed by Steve Loughran and Ryan Blue.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de8b6ca5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de8b6ca5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de8b6ca5
Branch: refs/heads/YARN-5881
Commit: de8b6ca5ef8614de6d6277b7617e27c788b0555c
Parents: 782ba3b
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 22 15:28:12 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 22 15:28:12 2017 +0000
----------------------------------------------------------------------
.../dev-support/findbugsExcludeFile.xml | 7 +
.../apache/hadoop/fs/FSDataOutputStream.java | 9 +
.../apache/hadoop/fs/PathExistsException.java | 4 +-
.../org/apache/hadoop/fs/StorageStatistics.java | 5 +
.../apache/hadoop/util/JsonSerialization.java | 299 +++
.../src/main/resources/core-default.xml | 117 +-
.../hadoop/fs/contract/ContractTestUtils.java | 51 +-
.../apache/hadoop/test/GenericTestUtils.java | 29 +-
.../org/apache/hadoop/test/HadoopTestBase.java | 51 +-
.../org/apache/hadoop/test/LambdaTestUtils.java | 144 +-
.../hadoop/util/TestJsonSerialization.java | 185 ++
.../mapreduce/TestMapreduceConfigFields.java | 27 +-
.../lib/output/BindingPathOutputCommitter.java | 184 ++
.../lib/output/FileOutputCommitter.java | 12 +-
.../lib/output/FileOutputCommitterFactory.java | 38 +
.../mapreduce/lib/output/FileOutputFormat.java | 10 +-
.../lib/output/NamedCommitterFactory.java | 79 +
.../lib/output/PathOutputCommitter.java | 17 +
.../lib/output/PathOutputCommitterFactory.java | 204 ++
.../src/main/resources/mapred-default.xml | 22 +
.../lib/output/TestPathOutputCommitter.java | 24 +-
.../output/TestPathOutputCommitterFactory.java | 495 +++++
hadoop-tools/hadoop-aws/pom.xml | 46 +-
.../hadoop/fs/s3a/AWSBadRequestException.java | 42 +
.../hadoop/fs/s3a/AWSClientIOException.java | 3 +-
.../hadoop/fs/s3a/AWSNoResponseException.java | 31 +
.../hadoop/fs/s3a/AWSRedirectException.java | 38 +
.../fs/s3a/AWSServiceThrottledException.java | 42 +
.../hadoop/fs/s3a/AWSStatus500Exception.java | 37 +
.../s3a/BlockingThreadPoolExecutorService.java | 2 +-
.../org/apache/hadoop/fs/s3a/Constants.java | 72 +-
.../fs/s3a/InconsistentAmazonS3Client.java | 232 ++-
.../java/org/apache/hadoop/fs/s3a/Invoker.java | 485 +++++
.../java/org/apache/hadoop/fs/s3a/Listing.java | 26 +-
.../java/org/apache/hadoop/fs/s3a/Retries.java | 92 +
.../hadoop/fs/s3a/S3ABlockOutputStream.java | 307 +--
.../org/apache/hadoop/fs/s3a/S3ADataBlocks.java | 2 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 940 +++++----
.../apache/hadoop/fs/s3a/S3AInputStream.java | 56 +-
.../hadoop/fs/s3a/S3AInstrumentation.java | 231 ++-
.../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 246 +++
.../hadoop/fs/s3a/S3AStorageStatistics.java | 12 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 324 ++-
.../org/apache/hadoop/fs/s3a/S3ListRequest.java | 11 +
.../hadoop/fs/s3a/S3ObjectAttributes.java | 10 +-
.../org/apache/hadoop/fs/s3a/Statistic.java | 56 +-
.../hadoop/fs/s3a/WriteOperationHelper.java | 474 +++++
.../fs/s3a/commit/AbstractS3ACommitter.java | 756 +++++++
.../s3a/commit/AbstractS3ACommitterFactory.java | 90 +
.../hadoop/fs/s3a/commit/CommitConstants.java | 240 +++
.../hadoop/fs/s3a/commit/CommitOperations.java | 596 ++++++
.../hadoop/fs/s3a/commit/CommitUtils.java | 129 ++
.../hadoop/fs/s3a/commit/CommitUtilsWithMR.java | 192 ++
.../apache/hadoop/fs/s3a/commit/Duration.java | 60 +
.../hadoop/fs/s3a/commit/DurationInfo.java | 59 +
.../s3a/commit/InternalCommitterConstants.java | 100 +
.../hadoop/fs/s3a/commit/LocalTempDir.java | 80 +
.../fs/s3a/commit/MagicCommitIntegration.java | 182 ++
.../hadoop/fs/s3a/commit/MagicCommitPaths.java | 229 ++
.../fs/s3a/commit/PathCommitException.java | 43 +
.../apache/hadoop/fs/s3a/commit/PutTracker.java | 100 +
.../fs/s3a/commit/S3ACommitterFactory.java | 129 ++
.../org/apache/hadoop/fs/s3a/commit/Tasks.java | 410 ++++
.../hadoop/fs/s3a/commit/ValidationFailure.java | 53 +
.../hadoop/fs/s3a/commit/files/PendingSet.java | 192 ++
.../s3a/commit/files/PersistentCommitData.java | 69 +
.../s3a/commit/files/SinglePendingCommit.java | 432 ++++
.../hadoop/fs/s3a/commit/files/SuccessData.java | 322 +++
.../fs/s3a/commit/files/package-info.java | 45 +
.../fs/s3a/commit/magic/MagicCommitTracker.java | 161 ++
.../s3a/commit/magic/MagicS3GuardCommitter.java | 288 +++
.../magic/MagicS3GuardCommitterFactory.java | 47 +
.../fs/s3a/commit/magic/package-info.java | 27 +
.../hadoop/fs/s3a/commit/package-info.java | 28 +
.../s3a/commit/staging/ConflictResolution.java | 33 +
.../staging/DirectoryStagingCommitter.java | 116 ++
.../DirectoryStagingCommitterFactory.java | 48 +
.../staging/PartitionedStagingCommitter.java | 159 ++
.../PartitionedStagingCommitterFactory.java | 48 +
.../hadoop/fs/s3a/commit/staging/Paths.java | 300 +++
.../fs/s3a/commit/staging/StagingCommitter.java | 851 ++++++++
.../staging/StagingCommitterConstants.java | 64 +
.../commit/staging/StagingCommitterFactory.java | 49 +
.../fs/s3a/commit/staging/package-info.java | 27 +
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 368 ++--
.../apache/hadoop/fs/s3a/s3guard/S3Guard.java | 2 +
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 16 +-
.../tools/hadoop-aws/committer_architecture.md | 1951 ++++++++++++++++++
.../markdown/tools/hadoop-aws/committers.md | 819 ++++++++
.../src/site/markdown/tools/hadoop-aws/index.md | 188 +-
.../site/markdown/tools/hadoop-aws/s3guard.md | 7 +-
.../site/markdown/tools/hadoop-aws/testing.md | 60 +
.../tools/hadoop-aws/troubleshooting_s3a.md | 124 ++
.../s3a/ITestS3AContractGetFileStatus.java | 11 +
.../hadoop/fs/s3a/AbstractS3AMockTest.java | 5 +-
.../hadoop/fs/s3a/AbstractS3ATestBase.java | 23 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 8 +-
.../hadoop/fs/s3a/ITestS3AEncryptionSSEC.java | 6 +-
.../hadoop/fs/s3a/ITestS3AFailureHandling.java | 49 +-
.../fs/s3a/ITestS3AFileOperationCost.java | 2 +
.../fs/s3a/ITestS3ATemporaryCredentials.java | 15 +-
.../fs/s3a/ITestS3GuardListConsistency.java | 45 +-
.../apache/hadoop/fs/s3a/MockS3AFileSystem.java | 322 +++
.../hadoop/fs/s3a/MockS3ClientFactory.java | 8 +
.../apache/hadoop/fs/s3a/S3ATestConstants.java | 6 +
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 132 +-
.../hadoop/fs/s3a/StorageStatisticsTracker.java | 99 +
.../org/apache/hadoop/fs/s3a/TestInvoker.java | 460 +++++
.../org/apache/hadoop/fs/s3a/TestListing.java | 2 +-
.../fs/s3a/TestS3AExceptionTranslation.java | 84 +-
.../fs/s3a/commit/AbstractCommitITest.java | 412 ++++
.../fs/s3a/commit/AbstractITCommitMRJob.java | 324 +++
.../fs/s3a/commit/AbstractITCommitProtocol.java | 1371 ++++++++++++
.../fs/s3a/commit/CommitterFaultInjection.java | 44 +
.../s3a/commit/CommitterFaultInjectionImpl.java | 131 ++
.../fs/s3a/commit/ITestCommitOperations.java | 545 +++++
.../fs/s3a/commit/LoggingTextOutputFormat.java | 133 ++
.../fs/s3a/commit/MiniDFSClusterService.java | 79 +
.../fs/s3a/commit/TestMagicCommitPaths.java | 246 +++
.../apache/hadoop/fs/s3a/commit/TestTasks.java | 550 +++++
.../fs/s3a/commit/magic/ITMagicCommitMRJob.java | 70 +
.../commit/magic/ITestMagicCommitProtocol.java | 190 ++
.../commit/magic/ITestS3AHugeMagicCommits.java | 195 ++
.../commit/staging/MockedStagingCommitter.java | 98 +
.../staging/PartitionedCommitterForTesting.java | 58 +
.../fs/s3a/commit/staging/StagingTestBase.java | 724 +++++++
.../hadoop/fs/s3a/commit/staging/TestPaths.java | 127 ++
.../commit/staging/TestStagingCommitter.java | 696 +++++++
.../TestStagingDirectoryOutputCommitter.java | 138 ++
.../TestStagingPartitionedFileListing.java | 186 ++
.../TestStagingPartitionedJobCommit.java | 236 +++
.../TestStagingPartitionedTaskCommit.java | 237 +++
.../integration/ITDirectoryCommitMRJob.java | 33 +
.../integration/ITPartitionCommitMRJob.java | 33 +
.../integration/ITStagingCommitMRJob.java | 66 +
.../ITestDirectoryCommitProtocol.java | 131 ++
.../ITestPartitionedCommitProtocol.java | 139 ++
.../integration/ITestStagingCommitProtocol.java | 190 ++
.../s3guard/AbstractS3GuardToolTestBase.java | 23 +
.../s3a/s3guard/TestDynamoDBMetadataStore.java | 13 +-
.../fs/s3a/scale/AbstractSTestS3AHugeFiles.java | 191 +-
.../hadoop-aws/src/test/resources/core-site.xml | 9 +-
.../src/test/resources/log4j.properties | 42 +-
.../registry/client/binding/JsonSerDeser.java | 224 +-
144 files changed, 24108 insertions(+), 1172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
index 4bafd8e..c056d21 100644
--- a/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-common-project/hadoop-common/dev-support/findbugsExcludeFile.xml
@@ -446,4 +446,11 @@
<Method name="setInstance"/>
<Bug pattern="ME_ENUM_FIELD_SETTER"/>
</Match>
+
+ <!-- findbugs is complaining that a stream isn't closed. It will be. -->
+ <Match>
+ <Class name="org.apache.hadoop.util.JsonSerialization"/>
+ <Method name="save"/>
+ <Bug pattern="OBL_UNSATISFIED_OBLIGATION"/>
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
index 1d95cd3..5970373 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStream.java
@@ -101,6 +101,15 @@ public class FSDataOutputStream extends DataOutputStream
out.close(); // This invokes PositionCache.close()
}
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "FSDataOutputStream{");
+ sb.append("wrappedStream=").append(wrappedStream);
+ sb.append('}');
+ return sb.toString();
+ }
+
/**
* Get a reference to the wrapped output stream.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java
index ccc1f0c..cd9f70a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/PathExistsException.java
@@ -27,7 +27,7 @@ public class PathExistsException extends PathIOException {
super(path, "File exists");
}
- protected PathExistsException(String path, String error) {
+ public PathExistsException(String path, String error) {
super(path, error);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
index d987ad0..5a3d736 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/StorageStatistics.java
@@ -97,6 +97,11 @@ public abstract class StorageStatistics {
public long getValue() {
return value;
}
+
+ @Override
+ public String toString() {
+ return name + " = " + value;
+ }
}
private final String name;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
new file mode 100644
index 0000000..15f4fef
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JsonSerialization.java
@@ -0,0 +1,299 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Support for marshalling objects to and from JSON.
+ *
+ * It constructs an object mapper as an instance field.
+ * and synchronizes access to those methods
+ * which use the mapper.
+ *
+ * This class was extracted from
+ * {@code org.apache.hadoop.registry.client.binding.JsonSerDeser},
+ * which is now a subclass of this class.
+ * @param <T> Type to marshal.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class JsonSerialization<T> {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(JsonSerialization.class);
+ private static final String UTF_8 = "UTF-8";
+
+ private final Class<T> classType;
+ private final ObjectMapper mapper;
+
+ /**
+ * Create an instance bound to a specific type.
+ * @param classType class to marshall
+ * @param failOnUnknownProperties fail if an unknown property is encountered.
+ * @param pretty generate pretty (indented) output?
+ */
+ public JsonSerialization(Class<T> classType,
+ boolean failOnUnknownProperties, boolean pretty) {
+ Preconditions.checkArgument(classType != null, "null classType");
+ this.classType = classType;
+ this.mapper = new ObjectMapper();
+ mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES,
+ failOnUnknownProperties);
+ mapper.configure(SerializationFeature.INDENT_OUTPUT, pretty);
+ }
+
+ /**
+ * Get the simple name of the class type to be marshalled.
+ * @return the name of the class being marshalled
+ */
+ public String getName() {
+ return classType.getSimpleName();
+ }
+
+ /**
+ * Convert from JSON.
+ *
+ * @param json input
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonParseException If the input is not well-formatted
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ @SuppressWarnings("unchecked")
+ public synchronized T fromJson(String json)
+ throws IOException, JsonParseException, JsonMappingException {
+ if (json.isEmpty()) {
+ throw new EOFException("No data");
+ }
+ try {
+ return mapper.readValue(json, classType);
+ } catch (IOException e) {
+ LOG.error("Exception while parsing json : {}\n{}", e, json, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Read from an input stream.
+ * @param stream stream to read from
+ * @return the parsed entity
+ * @throws IOException IO problems
+ * @throws JsonParseException If the input is not well-formatted
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ public synchronized T fromJsonStream(InputStream stream) throws IOException {
+ return mapper.readValue(stream, classType);
+ }
+
+ /**
+ * Load from a JSON text file.
+ * @param jsonFile input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonParseException If the input is not well-formatted
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ @SuppressWarnings("unchecked")
+ public synchronized T load(File jsonFile)
+ throws IOException, JsonParseException, JsonMappingException {
+ if (!jsonFile.isFile()) {
+ throw new FileNotFoundException("Not a file: " + jsonFile);
+ }
+ if (jsonFile.length() == 0) {
+ throw new EOFException("File is empty: " + jsonFile);
+ }
+ try {
+ return mapper.readValue(jsonFile, classType);
+ } catch (IOException e) {
+ LOG.error("Exception while parsing json file {}", jsonFile, e);
+ throw e;
+ }
+ }
+
+ /**
+ * Save to a local file. Any existing file is overwritten unless
+ * the OS blocks that.
+ * @param file file
+ * @param path path
+ * @throws IOException IO exception
+ */
+ public void save(File file, T instance) throws
+ IOException {
+ writeJsonAsBytes(instance, new FileOutputStream(file));
+ }
+
+ /**
+ * Convert from a JSON file.
+ * @param resource input file
+ * @return the parsed JSON
+ * @throws IOException IO problems
+ * @throws JsonParseException If the input is not well-formatted
+ * @throws JsonMappingException failure to map from the JSON to this class
+ */
+ @SuppressWarnings({"IOResourceOpenedButNotSafelyClosed"})
+ public synchronized T fromResource(String resource)
+ throws IOException, JsonParseException, JsonMappingException {
+ try (InputStream resStream = this.getClass()
+ .getResourceAsStream(resource)) {
+ if (resStream == null) {
+ throw new FileNotFoundException(resource);
+ }
+ return mapper.readValue(resStream, classType);
+ } catch (IOException e) {
+ LOG.error("Exception while parsing json resource {}", resource, e);
+ throw e;
+ }
+ }
+
+ /**
+ * clone by converting to JSON and back again.
+ * This is much less efficient than any Java clone process.
+ * @param instance instance to duplicate
+ * @return a new instance
+ * @throws IOException IO problems.
+ */
+ public T fromInstance(T instance) throws IOException {
+ return fromJson(toJson(instance));
+ }
+
+ /**
+ * Load from a Hadoop filesystem.
+ * There's a check for data availability after the file is open, by
+ * raising an EOFException if stream.available == 0.
+ * This allows for a meaningful exception without the round trip overhead
+ * of a getFileStatus call before opening the file. It may be brittle
+ * against an FS stream which doesn't return a value here, but the
+ * standard filesystems all do.
+ * JSON parsing and mapping problems
+ * are converted to IOEs.
+ * @param fs filesystem
+ * @param path path
+ * @return a loaded object
+ * @throws IOException IO or JSON parse problems
+ */
+ public T load(FileSystem fs, Path path) throws IOException {
+ try (FSDataInputStream dataInputStream = fs.open(path)) {
+ // throw an EOF exception if there is no data available.
+ if (dataInputStream.available() == 0) {
+ throw new EOFException("No data in " + path);
+ }
+ return fromJsonStream(dataInputStream);
+ } catch (JsonProcessingException e) {
+ throw new IOException(
+ String.format("Failed to read JSON file \"%s\": %s", path, e),
+ e);
+ }
+ }
+
+ /**
+ * Save to a Hadoop filesystem.
+ * @param fs filesystem
+ * @param path path
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public void save(FileSystem fs, Path path, T instance,
+ boolean overwrite) throws
+ IOException {
+ writeJsonAsBytes(instance, fs.create(path, overwrite));
+ }
+
+ /**
+ * Write the JSON as bytes, then close the file.
+ * @param dataOutputStream an output stream that will always be closed
+ * @throws IOException on any failure
+ */
+ private void writeJsonAsBytes(T instance,
+ OutputStream dataOutputStream) throws IOException {
+ try {
+ dataOutputStream.write(toBytes(instance));
+ } finally {
+ dataOutputStream.close();
+ }
+ }
+
+ /**
+ * Convert JSON to bytes.
+ * @param instance instance to convert
+ * @return a byte array
+ * @throws IOException IO problems
+ */
+ public byte[] toBytes(T instance) throws IOException {
+ return mapper.writeValueAsBytes(instance);
+ }
+
+ /**
+ * Deserialize from a byte array.
+ * @param bytes byte array
+ * @throws IOException IO problems
+ * @throws EOFException not enough data
+ */
+ public T fromBytes(byte[] bytes) throws IOException {
+ return fromJson(new String(bytes, 0, bytes.length, UTF_8));
+ }
+
+ /**
+ * Convert an instance to a JSON string.
+ * @param instance instance to convert
+ * @return a JSON string description
+ * @throws JsonProcessingException Json generation problems
+ */
+ public synchronized String toJson(T instance) throws JsonProcessingException {
+ return mapper.writeValueAsString(instance);
+ }
+
+ /**
+ * Convert an instance to a string form for output. This is a robust
+ * operation which will convert any JSON-generating exceptions into
+ * error text.
+ * @param instance non-null instance
+ * @return a JSON string
+ */
+ public String toString(T instance) {
+ Preconditions.checkArgument(instance != null, "Null instance argument");
+ try {
+ return toJson(instance);
+ } catch (JsonProcessingException e) {
+ return "Failed to convert to a string: " + e;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 8db9f44..183faa5 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1123,7 +1123,8 @@
<property>
<name>fs.s3a.multipart.purge.age</name>
<value>86400</value>
- <description>Minimum age in seconds of multipart uploads to purge.
+ <description>Minimum age in seconds of multipart uploads to purge
+ on startup if "fs.s3a.multipart.purge" is true
</description>
</property>
@@ -1345,6 +1346,120 @@
</property>
<property>
+ <name>fs.s3a.retry.limit</name>
+ <value>${fs.s3a.attempts.maximum}</value>
+ <description>
+ Number of times to retry any repeatable S3 client request on failure,
+ excluding throttling requests.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.interval</name>
+ <value>500ms</value>
+ <description>
+ Interval between attempts to retry operations for any reason other
+ than S3 throttle errors.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.throttle.limit</name>
+ <value>${fs.s3a.attempts.maximum}</value>
+ <description>
+ Number of times to retry any throttled request.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.throttle.interval</name>
+ <value>1000ms</value>
+ <description>
+ Interval between retry attempts on throttled requests.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.name</name>
+ <value>file</value>
+ <description>
+ Committer to create for output to S3A, one of:
+ "file", "directory", "partitioned", "magic".
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.magic.enabled</name>
+ <value>false</value>
+ <description>
+ Enable support in the filesystem for the S3 "Magic" committer.
+ When working with AWS S3, S3Guard must be enabled for the destination
+ bucket, as consistent metadata listings are required.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.threads</name>
+ <value>8</value>
+ <description>
+ Number of threads in committers for parallel operations on files
+ (upload, commit, abort, delete...)
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.tmp.path</name>
+ <value>tmp/staging</value>
+ <description>
+ Path in the cluster filesystem for temporary data.
+ This is for HDFS, not the local filesystem.
+ It is only for the summary data of each file, not the actual
+ data being committed.
+ Using an unqualified path guarantees that the full path will be
+ generated relative to the home directory of the user creating the job,
+ hence private (assuming home directory permissions are secure).
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.unique-filenames</name>
+ <value>true</value>
+ <description>
+ Option for final files to have a unique name through job attempt info,
+ or the value of fs.s3a.committer.staging.uuid
+ When writing data with the "append" conflict option, this guarantees
+ that new data will not overwrite any existing data.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.conflict-mode</name>
+ <value>fail</value>
+ <description>
+ Staging committer conflict resolution policy.
+ Supported: "fail", "append", "replace".
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.abort.pending.uploads</name>
+ <value>true</value>
+ <description>
+ Should the staging committers abort all pending uploads to the destination
+ directory?
+
+ Changing this if more than one partitioned committer is
+ writing to the same destination tree simultaneously; otherwise
+ the first job to complete will cancel all outstanding uploads from the
+ others. However, it may lead to leaked outstanding uploads from failed
+ tasks. If disabled, configure the bucket lifecycle to remove uploads
+ after a time period, and/or set up a workflow to explicitly delete
+ entries. Otherwise there is a risk that uncommitted uploads may run up
+ bills.
+ </description>
+</property>
+
+<property>
<name>fs.AbstractFileSystem.s3a.impl</name>
<value>org.apache.hadoop.fs.s3a.S3A</value>
<description>The implementation class of the S3A AbstractFileSystem.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
index e0cc7d6..d2cbca0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/ContractTestUtils.java
@@ -725,6 +725,8 @@ public class ContractTestUtils extends Assert {
/**
* Read in "length" bytes, convert to an ascii string.
+ * This uses {@link #toChar(byte)} to escape bytes, so cannot be used
+ * for round trip operations.
* @param fs filesystem
* @param path path to read
* @param length #of bytes to read.
@@ -742,6 +744,28 @@ public class ContractTestUtils extends Assert {
}
/**
+ * Read in "length" bytes, convert to UTF8 string.
+ * @param fs filesystem
+ * @param path path to read
+ * @param length #of bytes to read. If -1: use file length.
+ * @return the bytes read and converted to a string
+ * @throws IOException IO problems
+ */
+ public static String readUTF8(FileSystem fs,
+ Path path,
+ int length) throws IOException {
+ if (length < 0) {
+ FileStatus status = fs.getFileStatus(path);
+ length = (int) status.getLen();
+ }
+ try (FSDataInputStream in = fs.open(path)) {
+ byte[] buf = new byte[length];
+ in.readFully(0, buf);
+ return new String(buf, "UTF-8");
+ }
+ }
+
+ /**
* Take an array of filestats and convert to a string
* (prefixed with/ a [%02d] counter).
* @param stats array of stats
@@ -857,11 +881,30 @@ public class ContractTestUtils extends Assert {
*/
public static void assertPathExists(FileSystem fileSystem, String message,
Path path) throws IOException {
- if (!fileSystem.exists(path)) {
+ verifyPathExists(fileSystem, message, path);
+ }
+
+ /**
+ * Verify that a path exists, returning the file status of the path.
+ *
+ * @param fileSystem filesystem to examine
+ * @param message message to include in the assertion failure message
+ * @param path path in the filesystem
+ * @throws FileNotFoundException raised if the path is missing
+ * @throws IOException IO problems
+ */
+ public static FileStatus verifyPathExists(FileSystem fileSystem,
+ String message,
+ Path path) throws IOException {
+ try {
+ return fileSystem.getFileStatus(path);
+ } catch (FileNotFoundException e) {
//failure, report it
- ls(fileSystem, path.getParent());
- throw new FileNotFoundException(message + ": not found " + path
- + " in " + path.getParent());
+ LOG.error("{}: not found {}; parent listing is:\n{}",
+ message, path, ls(fileSystem, path.getParent()));
+ throw (IOException)new FileNotFoundException(
+ message + ": not found " + path + " in " + path.getParent())
+ .initCause(e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 4cb9f8b..0db6c73 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -316,20 +316,37 @@ public abstract class GenericTestUtils {
/**
* Assert that an exception's <code>toString()</code> value
* contained the expected text.
- * @param string expected string
+ * @param expectedText expected string
* @param t thrown exception
* @throws AssertionError if the expected string is not found
*/
- public static void assertExceptionContains(String string, Throwable t) {
+ public static void assertExceptionContains(String expectedText, Throwable t) {
+ assertExceptionContains(expectedText, t, "");
+ }
+
+ /**
+ * Assert that an exception's <code>toString()</code> value
+ * contained the expected text.
+ * @param expectedText expected string
+ * @param t thrown exception
+ * @param message any extra text for the string
+ * @throws AssertionError if the expected string is not found
+ */
+ public static void assertExceptionContains(String expectedText,
+ Throwable t,
+ String message) {
Assert.assertNotNull(E_NULL_THROWABLE, t);
String msg = t.toString();
if (msg == null) {
throw new AssertionError(E_NULL_THROWABLE_STRING, t);
}
- if (!msg.contains(string)) {
- throw new AssertionError("Expected to find '" + string + "' "
- + E_UNEXPECTED_EXCEPTION + ":"
- + StringUtils.stringifyException(t),
+ if (expectedText != null && !msg.contains(expectedText)) {
+ String prefix = org.apache.commons.lang.StringUtils.isEmpty(message)
+ ? "" : (message + ": ");
+ throw new AssertionError(
+ String.format("%s Expected to find '%s' %s: %s",
+ prefix, expectedText, E_UNEXPECTED_EXCEPTION,
+ StringUtils.stringifyException(t)),
t);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
index 43d5be8..cb7df4b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/HadoopTestBase.java
@@ -17,29 +17,37 @@
*/
package org.apache.hadoop.test;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Rule;
+import org.junit.rules.TestName;
import org.junit.rules.Timeout;
/**
* A base class for JUnit4 tests that sets a default timeout for all tests
- * that subclass this test
+ * that subclass this test.
+ *
+ * Threads are named to the method being executed, for ease of diagnostics
+ * in logs and thread dumps.
*/
-public abstract class HadoopTestBase {
+public abstract class HadoopTestBase extends Assert {
+
/**
- * System property name to set the test timeout: {@value}
+ * System property name to set the test timeout: {@value}.
*/
public static final String PROPERTY_TEST_DEFAULT_TIMEOUT =
- "test.default.timeout";
+ "test.default.timeout";
/**
* The default timeout (in milliseconds) if the system property
* {@link #PROPERTY_TEST_DEFAULT_TIMEOUT}
- * is not set: {@value}
+ * is not set: {@value}.
*/
public static final int TEST_DEFAULT_TIMEOUT_VALUE = 100000;
/**
- * The JUnit rule that sets the default timeout for tests
+ * The JUnit rule that sets the default timeout for tests.
*/
@Rule
public Timeout defaultTimeout = retrieveTestTimeout();
@@ -64,4 +72,35 @@ public abstract class HadoopTestBase {
}
return new Timeout(millis);
}
+
+ /**
+ * The method name.
+ */
+ @Rule
+ public TestName methodName = new TestName();
+
+ /**
+ * Get the method name; defaults to the value of {@link #methodName}.
+ * Subclasses may wish to override it, which will tune the thread naming.
+ * @return the name of the method.
+ */
+ protected String getMethodName() {
+ return methodName.getMethodName();
+ }
+
+ /**
+ * Static initializer names this thread "JUnit".
+ */
+ @BeforeClass
+ public static void nameTestThread() {
+ Thread.currentThread().setName("JUnit");
+ }
+
+ /**
+ * Before each method, the thread is renamed to match the method name.
+ */
+ @Before
+ public void nameThreadToMethod() {
+ Thread.currentThread().setName("JUnit-" + getMethodName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java
index 3ea9ab8..22208f7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/LambdaTestUtils.java
@@ -19,11 +19,13 @@
package org.apache.hadoop.test;
import com.google.common.base.Preconditions;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.util.Time;
+import java.util.Optional;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeoutException;
@@ -43,7 +45,7 @@ import java.util.concurrent.TimeoutException;
* check; useful when checking the contents of the exception.
*/
public final class LambdaTestUtils {
- public static final Logger LOG =
+ private static final Logger LOG =
LoggerFactory.getLogger(LambdaTestUtils.class);
private LambdaTestUtils() {
@@ -60,6 +62,7 @@ public final class LambdaTestUtils {
* Interface to implement for converting a timeout into some form
* of exception to raise.
*/
+ @FunctionalInterface
public interface TimeoutHandler {
/**
@@ -371,16 +374,11 @@ public final class LambdaTestUtils {
Class<E> clazz,
Callable<T> eval)
throws Exception {
- try {
- T result = eval.call();
- throw new AssertionError("Expected an exception, got "
- + robustToString(result));
- } catch (Throwable e) {
- if (clazz.isAssignableFrom(e.getClass())) {
- return (E)e;
- }
- throw e;
- }
+ return intercept(clazz,
+ null,
+ "Expected a " + clazz.getName() + " to be thrown," +
+ " but got the result: ",
+ eval);
}
/**
@@ -451,6 +449,59 @@ public final class LambdaTestUtils {
}
/**
+ * Intercept an exception; throw an {@code AssertionError} if one not raised.
+ * The caught exception is rethrown if it is of the wrong class or
+ * does not contain the text defined in {@code contained}.
+ * <p>
+ * Example: expect deleting a nonexistent file to raise a
+ * {@code FileNotFoundException} with the {@code toString()} value
+ * containing the text {@code "missing"}.
+ * <pre>
+ * FileNotFoundException ioe = intercept(FileNotFoundException.class,
+ * "missing",
+ * "path should not be found",
+ * () -> {
+ * filesystem.delete(new Path("/missing"), false);
+ * });
+ * </pre>
+ *
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param contained string which must be in the {@code toString()} value
+ * of the exception
+ * @param message any message tho include in exception/log messages
+ * @param eval expression to eval
+ * @param <T> return type of expression
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type and contents
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ * The error includes the {@code toString()} value of the result, if this
+ * can be determined.
+ * @see GenericTestUtils#assertExceptionContains(String, Throwable)
+ */
+ public static <T, E extends Throwable> E intercept(
+ Class<E> clazz,
+ String contained,
+ String message,
+ Callable<T> eval)
+ throws Exception {
+ E ex;
+ try {
+ T result = eval.call();
+ throw new AssertionError(message + ": " + robustToString(result));
+ } catch (Throwable e) {
+ if (!clazz.isAssignableFrom(e.getClass())) {
+ throw e;
+ } else {
+ ex = (E) e;
+ }
+ }
+ GenericTestUtils.assertExceptionContains(contained, ex, message);
+ return ex;
+ }
+
+ /**
* Variant of {@link #intercept(Class, Callable)} to simplify void
* invocations.
* @param clazz class of exception; the raised exception must be this class
@@ -468,9 +519,41 @@ public final class LambdaTestUtils {
String contained,
VoidCallable eval)
throws Exception {
- E ex = intercept(clazz, eval);
- GenericTestUtils.assertExceptionContains(contained, ex);
- return ex;
+ return intercept(clazz, contained,
+ "Expecting " + clazz.getName()
+ + (contained != null? (" with text " + contained) : "")
+ + " but got ",
+ () -> {
+ eval.call();
+ return "void";
+ });
+ }
+
+ /**
+ * Variant of {@link #intercept(Class, Callable)} to simplify void
+ * invocations.
+ * @param clazz class of exception; the raised exception must be this class
+ * <i>or a subclass</i>.
+ * @param contained string which must be in the {@code toString()} value
+ * of the exception
+ * @param message any message tho include in exception/log messages
+ * @param eval expression to eval
+ * @param <E> exception class
+ * @return the caught exception if it was of the expected type
+ * @throws Exception any other exception raised
+ * @throws AssertionError if the evaluation call didn't raise an exception.
+ */
+ public static <E extends Throwable> E intercept(
+ Class<E> clazz,
+ String contained,
+ String message,
+ VoidCallable eval)
+ throws Exception {
+ return intercept(clazz, contained, message,
+ () -> {
+ eval.call();
+ return "void";
+ });
}
/**
@@ -495,6 +578,38 @@ public final class LambdaTestUtils {
}
/**
+ * Assert that an optional value matches an expected one;
+ * checks include null and empty on the actual value.
+ * @param message message text
+ * @param expected expected value
+ * @param actual actual optional value
+ * @param <T> type
+ */
+ public static <T> void assertOptionalEquals(String message,
+ T expected,
+ Optional<T> actual) {
+ Assert.assertNotNull(message, actual);
+ Assert.assertTrue(message +" -not present", actual.isPresent());
+ Assert.assertEquals(message, expected, actual.get());
+ }
+
+ /**
+ * Assert that an optional value matches an expected one;
+ * checks include null and empty on the actual value.
+ * @param message message text
+ * @param expected expected value
+ * @param actual actual optional value
+ * @param <T> type
+ */
+ public static <T> void assertOptionalUnset(String message,
+ Optional<T> actual) {
+ Assert.assertNotNull(message, actual);
+ if (actual.isPresent()) {
+ Assert.fail("Expected empty option, got " + actual.get().toString());
+ }
+ }
+
+ /**
* Returns {@code TimeoutException} on a timeout. If
* there was a inner class passed in, includes it as the
* inner failure.
@@ -638,6 +753,7 @@ public final class LambdaTestUtils {
* A simple interface for lambdas, which returns nothing; this exists
* to simplify lambda tests on operations with no return value.
*/
+ @FunctionalInterface
public interface VoidCallable {
void call() throws Exception;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java
new file mode 100644
index 0000000..991697d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/TestJsonSerialization.java
@@ -0,0 +1,185 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.util;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.Serializable;
+import java.util.Objects;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.HadoopTestBase;
+import org.apache.hadoop.test.LambdaTestUtils;
+
+/**
+ * Test the JSON serialization helper.
+ */
+public class TestJsonSerialization extends HadoopTestBase {
+
+ private final JsonSerialization<KeyVal> serDeser =
+ new JsonSerialization<>(KeyVal.class, true, true);
+
+ private final KeyVal source = new KeyVal("key", "1");
+
+ private static class KeyVal implements Serializable {
+ private String name;
+ private String value;
+
+ KeyVal(String name, String value) {
+ this.name = name;
+ this.value = value;
+ }
+
+ KeyVal() {
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("SimpleJson{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", value='").append(value).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ KeyVal that = (KeyVal) o;
+ return Objects.equals(name, that.name) &&
+ Objects.equals(value, that.value);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(name, value);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public void setValue(String value) {
+ this.value = value;
+ }
+ }
+
+ @Test
+ public void testStringRoundTrip() throws Throwable {
+ String wire = serDeser.toJson(source);
+ KeyVal unmarshalled = serDeser.fromJson(wire);
+ assertEquals("Failed to unmarshall: " + wire, source, unmarshalled);
+ }
+
+ @Test
+ public void testBytesRoundTrip() throws Throwable {
+ byte[] wire = serDeser.toBytes(source);
+ KeyVal unmarshalled = serDeser.fromBytes(wire);
+ assertEquals(source, unmarshalled);
+ }
+
+ @Test
+ public void testBadBytesRoundTrip() throws Throwable {
+ LambdaTestUtils.intercept(JsonParseException.class,
+ "token",
+ () -> serDeser.fromBytes(new byte[]{'a'}));
+ }
+
+ @Test
+ public void testCloneViaJson() throws Throwable {
+ KeyVal unmarshalled = serDeser.fromInstance(source);
+ assertEquals(source, unmarshalled);
+ }
+
+ @Test
+ public void testFileRoundTrip() throws Throwable {
+ File tempFile = File.createTempFile("Keyval", ".json");
+ tempFile.delete();
+ try {
+ serDeser.save(tempFile, source);
+ assertEquals(source, serDeser.load(tempFile));
+ } finally {
+ tempFile.delete();
+ }
+ }
+
+ @Test
+ public void testEmptyFile() throws Throwable {
+ File tempFile = File.createTempFile("Keyval", ".json");
+ try {
+ LambdaTestUtils.intercept(EOFException.class,
+ "empty",
+ () -> serDeser.load(tempFile));
+ } finally {
+ tempFile.delete();
+ }
+ }
+
+ @Test
+ public void testFileSystemRoundTrip() throws Throwable {
+ File tempFile = File.createTempFile("Keyval", ".json");
+ tempFile.delete();
+ Path tempPath = new Path(tempFile.toURI());
+ LocalFileSystem fs = FileSystem.getLocal(new Configuration());
+ try {
+ serDeser.save(fs, tempPath, source, false);
+ assertEquals(source, serDeser.load(fs, tempPath));
+ } finally {
+ fs.delete(tempPath, false);
+ }
+ }
+
+ @Test
+ public void testFileSystemEmptyPath() throws Throwable {
+ File tempFile = File.createTempFile("Keyval", ".json");
+ Path tempPath = new Path(tempFile.toURI());
+ LocalFileSystem fs = FileSystem.getLocal(new Configuration());
+ try {
+ LambdaTestUtils.intercept(EOFException.class,
+ () -> serDeser.load(fs, tempPath));
+ fs.delete(tempPath, false);
+ LambdaTestUtils.intercept(FileNotFoundException.class,
+ () -> serDeser.load(fs, tempPath));
+ } finally {
+ fs.delete(tempPath, false);
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/TestMapreduceConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/TestMapreduceConfigFields.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/TestMapreduceConfigFields.java
index 5d42fbf..f8aaab7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/TestMapreduceConfigFields.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/TestMapreduceConfigFields.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
/**
@@ -53,14 +54,23 @@ public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
@SuppressWarnings("deprecation")
@Override
public void initializeMemberVariables() {
- xmlFilename = new String("mapred-default.xml");
- configurationClasses = new Class[] { MRJobConfig.class, MRConfig.class,
- JHAdminConfig.class, ShuffleHandler.class, FileOutputFormat.class,
- FileInputFormat.class, Job.class, NLineInputFormat.class,
- JobConf.class, FileOutputCommitter.class };
+ xmlFilename = "mapred-default.xml";
+ configurationClasses = new Class[] {
+ MRJobConfig.class,
+ MRConfig.class,
+ JHAdminConfig.class,
+ ShuffleHandler.class,
+ FileOutputFormat.class,
+ FileInputFormat.class,
+ Job.class,
+ NLineInputFormat.class,
+ JobConf.class,
+ FileOutputCommitter.class,
+ PathOutputCommitterFactory.class
+ };
// Initialize used variables
- configurationPropsToSkipCompare = new HashSet<String>();
+ configurationPropsToSkipCompare = new HashSet<>();
// Set error modes
errorIfMissingConfigProps = true;
@@ -82,6 +92,11 @@ public class TestMapreduceConfigFields extends TestConfigurationFieldsBase {
MRJobConfig.MAP_RESOURCE_TYPE_PREFIX);
configurationPropsToSkipCompare.add(
MRJobConfig.REDUCE_RESOURCE_TYPE_PREFIX);
+
+ // PathOutputCommitterFactory values
+ xmlPrefixToSkipCompare = new HashSet<>();
+ xmlPrefixToSkipCompare.add(
+ PathOutputCommitterFactory.COMMITTER_FACTORY_SCHEME);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java
new file mode 100644
index 0000000..f12678b
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/BindingPathOutputCommitter.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * This is a special committer which creates the factory for the committer and
+ * runs off that. Why does it exist? So that you can explicitly instantiate
+ * a committer by classname and yet still have the actual implementation
+ * driven dynamically by the factory options and destination filesystem.
+ * This simplifies integration
+ * with existing code which takes the classname of a committer.
+ * There's no factory for this, as that would lead to a loop.
+ *
+ * All commit protocol methods and accessors are delegated to the
+ * wrapped committer.
+ *
+ * How to use:
+ *
+ * <ol>
+ * <li>
+ * In applications which take a classname of committer in
+ * a configuration option, set it to the canonical name of this class
+ * (see {@link #NAME}). When this class is instantiated, it will
+ * use the factory mechanism to locate the configured committer for the
+ * destination.
+ * </li>
+ * <li>
+ * In code, explicitly create an instance of this committer through
+ * its constructor, then invoke commit lifecycle operations on it.
+ * The dynamically configured committer will be created in the constructor
+ * and have the lifecycle operations relayed to it.
+ * </li>
+ * </ol>
+ *
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class BindingPathOutputCommitter extends PathOutputCommitter {
+
+ /**
+ * The classname for use in configurations.
+ */
+ public static final String NAME
+ = BindingPathOutputCommitter.class.getCanonicalName();
+
+ /**
+ * The bound committer.
+ */
+ private final PathOutputCommitter committer;
+
+ /**
+ * Instantiate.
+ * @param outputPath output path (may be null)
+ * @param context task context
+ * @throws IOException on any failure.
+ */
+ public BindingPathOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ committer = PathOutputCommitterFactory.getCommitterFactory(outputPath,
+ context.getConfiguration())
+ .createOutputCommitter(outputPath, context);
+ }
+
+ @Override
+ public Path getOutputPath() {
+ return committer.getOutputPath();
+ }
+
+ @Override
+ public Path getWorkPath() throws IOException {
+ return committer.getWorkPath();
+ }
+
+ @Override
+ public void setupJob(JobContext jobContext) throws IOException {
+ committer.setupJob(jobContext);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskContext) throws IOException {
+ committer.setupTask(taskContext);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskContext)
+ throws IOException {
+ return committer.needsTaskCommit(taskContext);
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskContext) throws IOException {
+ committer.commitTask(taskContext);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskContext) throws IOException {
+ committer.abortTask(taskContext);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void cleanupJob(JobContext jobContext) throws IOException {
+ super.cleanupJob(jobContext);
+ }
+
+ @Override
+ public void commitJob(JobContext jobContext) throws IOException {
+ committer.commitJob(jobContext);
+ }
+
+ @Override
+ public void abortJob(JobContext jobContext, JobStatus.State state)
+ throws IOException {
+ committer.abortJob(jobContext, state);
+ }
+
+ @SuppressWarnings("deprecation")
+ @Override
+ public boolean isRecoverySupported() {
+ return committer.isRecoverySupported();
+ }
+
+ @Override
+ public boolean isCommitJobRepeatable(JobContext jobContext)
+ throws IOException {
+ return committer.isCommitJobRepeatable(jobContext);
+ }
+
+ @Override
+ public boolean isRecoverySupported(JobContext jobContext) throws IOException {
+ return committer.isRecoverySupported(jobContext);
+ }
+
+ @Override
+ public void recoverTask(TaskAttemptContext taskContext) throws IOException {
+ committer.recoverTask(taskContext);
+ }
+
+ @Override
+ public boolean hasOutputPath() {
+ return committer.hasOutputPath();
+ }
+
+ @Override
+ public String toString() {
+ return "BindingPathOutputCommitter{"
+ + "committer=" + committer +
+ '}';
+ }
+
+ /**
+ * Get the inner committer.
+ * @return the bonded committer.
+ */
+ public PathOutputCommitter getCommitter() {
+ return committer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
index 325b2e7..86af2cf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
@@ -155,17 +155,11 @@ public class FileOutputCommitter extends PathOutputCommitter {
* @return the path where final output of the job should be placed. This
* could also be considered the committed application attempt path.
*/
- private Path getOutputPath() {
+ @Override
+ public Path getOutputPath() {
return this.outputPath;
}
-
- /**
- * @return true if we have an output path set, else false.
- */
- private boolean hasOutputPath() {
- return this.outputPath != null;
- }
-
+
/**
* @return the path where the output of pending job attempts are
* stored.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitterFactory.java
new file mode 100644
index 0000000..12b2841
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitterFactory.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Creates a {@link FileOutputCommitter}, always.
+ */
+public final class FileOutputCommitterFactory
+ extends PathOutputCommitterFactory {
+
+ @Override
+ public PathOutputCommitter createOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return createFileOutputCommitter(outputPath, context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
index 0e7efa3..bbda26a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.java
@@ -328,12 +328,14 @@ public abstract class FileOutputFormat<K, V> extends OutputFormat<K, V> {
job.getConfiguration().set(BASE_OUTPUT_NAME, name);
}
- public synchronized
- OutputCommitter getOutputCommitter(TaskAttemptContext context
- ) throws IOException {
+ public synchronized
+ OutputCommitter getOutputCommitter(TaskAttemptContext context)
+ throws IOException {
if (committer == null) {
Path output = getOutputPath(context);
- committer = new FileOutputCommitter(output, context);
+ committer = PathOutputCommitterFactory.getCommitterFactory(
+ output,
+ context.getConfiguration()).createOutputCommitter(output, context);
}
return committer;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java
new file mode 100644
index 0000000..b7378af
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/NamedCommitterFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output;
+
+import java.io.IOException;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * A factory which creates any named committer identified
+ * in the option {@link PathOutputCommitterFactory#NAMED_COMMITTER_CLASS}.
+ */
+public final class NamedCommitterFactory extends
+ PathOutputCommitterFactory {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NamedCommitterFactory.class);
+
+ @SuppressWarnings("JavaReflectionMemberAccess")
+ @Override
+ public PathOutputCommitter createOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ Class<? extends PathOutputCommitter> clazz = loadCommitterClass(context);
+ LOG.debug("Using PathOutputCommitter implementation {}", clazz);
+ try {
+ Constructor<? extends PathOutputCommitter> ctor
+ = clazz.getConstructor(Path.class, TaskAttemptContext.class);
+ return ctor.newInstance(outputPath, context);
+ } catch (NoSuchMethodException
+ | InstantiationException
+ | IllegalAccessException
+ | InvocationTargetException e) {
+ throw new IOException("Failed to create " + clazz
+ + ":" + e, e);
+ }
+ }
+
+ /**
+ * Load the class named in {@link #NAMED_COMMITTER_CLASS}.
+ * @param context job or task context
+ * @return the committer class
+ * @throws IOException if no committer was defined.
+ */
+ private Class<? extends PathOutputCommitter> loadCommitterClass(
+ JobContext context) throws IOException {
+ Preconditions.checkNotNull(context, "null context");
+ Configuration conf = context.getConfiguration();
+ String value = conf.get(NAMED_COMMITTER_CLASS, "");
+ if (value.isEmpty()) {
+ throw new IOException("No committer defined in " + NAMED_COMMITTER_CLASS);
+ }
+ return conf.getClass(NAMED_COMMITTER_CLASS,
+ FileOutputCommitter.class, PathOutputCommitter.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
index 2df30ba..3679d9f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitter.java
@@ -76,9 +76,26 @@ public abstract class PathOutputCommitter extends OutputCommitter {
}
/**
+ * Get the final directory where work will be placed once the job
+ * is committed. This may be null, in which case, there is no output
+ * path to write data to.
+ * @return the path where final output of the job should be placed.
+ */
+ public abstract Path getOutputPath();
+
+ /**
+ * Predicate: is there an output path?
+ * @return true if we have an output path set, else false.
+ */
+ public boolean hasOutputPath() {
+ return getOutputPath() != null;
+ }
+
+ /**
* Get the directory that the task should write results into.
* Warning: there's no guarantee that this work path is on the same
* FS as the final output, or that it's visible across machines.
+ * May be null.
* @return the work directory
* @throws IOException IO problem
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
new file mode 100644
index 0000000..0df14d1
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/PathOutputCommitterFactory.java
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * A factory for committers implementing the {@link PathOutputCommitter}
+ * methods, and so can be used from {@link FileOutputFormat}.
+ * The base implementation returns {@link FileOutputCommitter} instances.
+ *
+ * Algorithm:
+ * <ol>
+ * <ul>If an explicit committer factory is named, it is used.</ul>
+ * <ul>The output path is examined.
+ * If is non null and there is an explicit schema for that filesystem,
+ * its factory is instantiated.</ul>
+ * <ul>Otherwise, an instance of {@link FileOutputCommitter} is
+ * created.</ul>
+ * </ol>
+ *
+ * In {@link FileOutputFormat}, the created factory has its method
+ * {@link #createOutputCommitter(Path, TaskAttemptContext)} with a task
+ * attempt context and a possibly null path.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class PathOutputCommitterFactory extends Configured {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PathOutputCommitterFactory.class);
+
+ /**
+ * Name of the configuration option used to configure the
+ * output committer factory to use unless there is a specific
+ * one for a schema.
+ */
+ public static final String COMMITTER_FACTORY_CLASS =
+ "mapreduce.outputcommitter.factory.class";
+
+ /**
+ * Scheme prefix for per-filesystem scheme committers.
+ */
+ public static final String COMMITTER_FACTORY_SCHEME =
+ "mapreduce.outputcommitter.factory.scheme";
+
+ /**
+ * String format pattern for per-filesystem scheme committers.
+ */
+ public static final String COMMITTER_FACTORY_SCHEME_PATTERN =
+ COMMITTER_FACTORY_SCHEME + ".%s";
+
+
+ /**
+ * The {@link FileOutputCommitter} factory.
+ */
+ public static final String FILE_COMMITTER_FACTORY =
+ "org.apache.hadoop.mapreduce.lib.output.FileOutputCommitterFactory";
+
+ /**
+ * The {@link FileOutputCommitter} factory.
+ */
+ public static final String NAMED_COMMITTER_FACTORY =
+ "org.apache.hadoop.mapreduce.lib.output.NamedCommitterFactory";
+
+ /**
+ * The named output committer.
+ * Creates any committer listed in
+ */
+ public static final String NAMED_COMMITTER_CLASS =
+ "mapreduce.outputcommitter.named.classname";
+
+ /**
+ * Default committer factory name: {@value}.
+ */
+ public static final String COMMITTER_FACTORY_DEFAULT =
+ FILE_COMMITTER_FACTORY;
+
+ /**
+ * Create an output committer for a task attempt.
+ * @param outputPath output path. This may be null.
+ * @param context context
+ * @return a new committer
+ * @throws IOException problems instantiating the committer
+ */
+ public PathOutputCommitter createOutputCommitter(
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return createFileOutputCommitter(outputPath, context);
+ }
+
+ /**
+ * Create an instance of the default committer, a {@link FileOutputCommitter}
+ * for a task.
+ * @param outputPath the task's output path, or or null if no output path
+ * has been defined.
+ * @param context the task attempt context
+ * @return the committer to use
+ * @throws IOException problems instantiating the committer
+ */
+ protected final PathOutputCommitter createFileOutputCommitter(
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ LOG.debug("Creating FileOutputCommitter for path {} and context {}",
+ outputPath, context);
+ return new FileOutputCommitter(outputPath, context);
+ }
+
+ /**
+ * Get the committer factory for a configuration.
+ * @param outputPath the job's output path. If null, it means that the
+ * schema is unknown and a per-schema factory cannot be determined.
+ * @param conf configuration
+ * @return an instantiated committer factory
+ */
+ public static PathOutputCommitterFactory getCommitterFactory(
+ Path outputPath,
+ Configuration conf) {
+ // determine which key to look up the overall one or a schema-specific
+ // key
+ LOG.debug("Looking for committer factory for path {}", outputPath);
+ String key = COMMITTER_FACTORY_CLASS;
+ if (StringUtils.isEmpty(conf.getTrimmed(key)) && outputPath != null) {
+ // there is no explicit factory and there's an output path
+ // Get the scheme of the destination
+ String scheme = outputPath.toUri().getScheme();
+
+ // and see if it has a key
+ String schemeKey = String.format(COMMITTER_FACTORY_SCHEME_PATTERN,
+ scheme);
+ if (StringUtils.isNotEmpty(conf.getTrimmed(schemeKey))) {
+ // it does, so use that key in the classname lookup
+ LOG.debug("Using schema-specific factory for {}", outputPath);
+ key = schemeKey;
+ } else {
+ LOG.debug("No scheme-specific factory defined in {}", schemeKey);
+ }
+ }
+
+ // create the factory. Before using Configuration.getClass, check
+ // for an empty configuration value, as that raises ClassNotFoundException.
+ Class<? extends PathOutputCommitterFactory> factory;
+ String trimmedValue = conf.getTrimmed(key, "");
+ if (StringUtils.isEmpty(trimmedValue)) {
+ // empty/null value, use default
+ LOG.debug("No output committer factory defined,"
+ + " defaulting to FileOutputCommitterFactory");
+ factory = FileOutputCommitterFactory.class;
+ } else {
+ // key is set, get the class
+ factory = conf.getClass(key,
+ FileOutputCommitterFactory.class,
+ PathOutputCommitterFactory.class);
+ LOG.debug("Using OutputCommitter factory class {} from key {}",
+ factory, key);
+ }
+ return ReflectionUtils.newInstance(factory, conf);
+ }
+
+ /**
+ * Create the committer factory for a task attempt & destination, then
+ * create the committer from it.
+ * @param outputPath the task's output path, or or null if no output path
+ * has been defined.
+ * @param context the task attempt context
+ * @return the committer to use
+ * @throws IOException problems instantiating the committer
+ */
+ public static PathOutputCommitter createCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return getCommitterFactory(outputPath,
+ context.getConfiguration())
+ .createOutputCommitter(outputPath, context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 9d166c7..1e432ce 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -2043,4 +2043,26 @@
<name>mapreduce.job.send-token-conf</name>
<value></value>
</property>
+
+<property>
+ <description>
+ The name of an output committer factory for MRv2 FileOutputFormat to use
+ for committing work. If set, overrides any per-filesystem committer
+ defined for the destination filesystem.
+ </description>
+ <name>mapreduce.outputcommitter.factory.class</name>
+ <value></value>
+</property>
+
+
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.s3a</name>
+ <value>org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory</value>
+ <description>
+ The committer factory to use when writing data to S3A filesystems.
+ If mapreduce.outputcommitter.factory.class is set, it will
+ override this property.
+ </description>
+</property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitter.java
index 9cff82f..3b73934 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitter.java
@@ -109,14 +109,34 @@ public class TestPathOutputCommitter extends Assert {
public void abortTask(TaskAttemptContext taskContext) throws IOException {
}
+
+ @Override
+ public Path getOutputPath() {
+ return null;
+ }
}
/**
* Stub task context.
+ * The {@link #getConfiguration()} method returns the configuration supplied
+ * in the constructor; while {@link #setOutputCommitter(OutputCommitter)}
+ * sets the committer returned in {@link #getOutputCommitter()}.
+ * Otherwise, the methods are all no-ops.
*/
- public class TaskContext
+ public static class TaskContext
implements TaskInputOutputContext<String, String, String, String> {
+ private final Configuration configuration;
+
+ public TaskContext() {
+ this(new Configuration());
+ }
+
+ public TaskContext(Configuration conf) {
+ this.configuration = conf;
+ }
+
+
private OutputCommitter outputCommitter;
public void setOutputCommitter(OutputCommitter outputCommitter) {
@@ -180,7 +200,7 @@ public class TestPathOutputCommitter extends Assert {
@Override
public Configuration getConfiguration() {
- return null;
+ return configuration;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/49] hadoop git commit: YARN-7330. Add support to show GPU in UI
including metrics. Contributed by Wangda Tan.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
deleted file mode 100644
index 1e8549b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node.hbs
+++ /dev/null
@@ -1,125 +0,0 @@
-{{!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
---}}
-
-{{breadcrumb-bar breadcrumbs=breadcrumbs}}
-
-<div class="col-md-12 container-fluid">
- <div class="row">
-
- {{node-menu-panel path="yarn-node" nodeId=model.rmNode.id nodeAddr=model.node.id}}
-
- <div class="col-md-10 container-fluid">
-
- <div class="row">
- <div class="col-md-12 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">Node Information: {{model.rmNode.id}}</div>
- <table class="table">
- <tbody>
- <tr>
- <td>Total Vmem allocated for Containers</td>
- <td>{{divide num=model.node.totalVmemAllocatedContainersMB den=1024}} GB</td>
- </tr>
- <tr>
- <td>Vmem enforcement enabled</td>
- <td>{{model.node.vmemCheckEnabled}}</td>
- </tr>
- <tr>
- <td>Total Pmem allocated for Containers</td>
- <td>{{divide num=model.node.totalPmemAllocatedContainersMB den=1024}} GB</td>
- </tr>
- <tr>
- <td>Pmem enforcement enabled</td>
- <td>{{model.node.pmemCheckEnabled}}</td>
- </tr>
- <tr>
- <td>Total VCores allocated for Containers</td>
- <td>{{model.node.totalVCoresAllocatedContainers}}</td>
- </tr>
- <tr>
- <td>Node Healthy Status</td>
- <td>{{model.node.nodeHealthy}}</td>
- </tr>
- <tr>
- <td>Last Node Health Report Time</td>
- <td>{{model.node.lastNodeUpdateTime}}</td>
- </tr>
- <tr>
- <td>Node Health Report</td>
- <td>{{model.node.healthReport}}</td>
- </tr>
- {{#if model.node.nmStartupTime}}
- <tr>
- <td>Node Manager Start Time</td>
- <td>{{model.node.nmStartupTime}}</td>
- </tr>
- {{/if}}
- <tr>
- <td>Node Manager Version</td>
- <td>{{model.node.nodeManagerBuildVersion}}</td>
- </tr>
- <tr>
- <td>Hadoop Version</td>
- <td>{{model.node.hadoopBuildVersion}}</td>
- </tr>
- </tbody>
- </table>
- </div>
- </div>
- </div>
-
- <div class="row">
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - Memory
- </div>
- <div class="container-fluid" id="mem-donut-chart">
- {{donut-chart data=model.rmNode.getMemoryDataForDonutChart
- showLabels=true
- parentId="mem-donut-chart"
- ratio=0.6
- type="memory"
- colorTargets="good"
- colorTargetReverse=true
- maxHeight=350}}
- </div>
- </div>
- </div>
-
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - VCores
- </div>
- <div class="container-fluid" id="vcore-donut-chart">
- {{donut-chart data=model.rmNode.getVCoreDataForDonutChart
- showLabels=true
- parentId="vcore-donut-chart"
- ratio=0.6
- colorTargets="good"
- colorTargetReverse=true
- maxHeight=350}}
- </div>
- </div>
- </div>
- </div>
- </div>
- </div>
-</div>
-{{outlet}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs
new file mode 100644
index 0000000..ad411c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/info.hbs
@@ -0,0 +1,154 @@
+{{!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--}}
+
+{{breadcrumb-bar breadcrumbs=breadcrumbs}}
+
+<div class="col-md-12 container-fluid">
+ <div class="row">
+
+ {{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
+ nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
+
+ <div class="col-md-10 container-fluid">
+
+ <div class="row">
+ <div class="col-md-12 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">Node
+ Information: {{model.rmNode.id}}</div>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Total Vmem allocated for Containers</td>
+ <td>{{divide num=model.node.totalVmemAllocatedContainersMB
+ den=1024}} GB
+ </td>
+ </tr>
+ <tr>
+ <td>Vmem enforcement enabled</td>
+ <td>{{model.node.vmemCheckEnabled}}</td>
+ </tr>
+ <tr>
+ <td>Total Pmem allocated for Containers</td>
+ <td>{{divide num=model.node.totalPmemAllocatedContainersMB
+ den=1024}} GB
+ </td>
+ </tr>
+ <tr>
+ <td>Pmem enforcement enabled</td>
+ <td>{{model.node.pmemCheckEnabled}}</td>
+ </tr>
+ <tr>
+ <td>Total VCores allocated for Containers</td>
+ <td>{{model.node.totalVCoresAllocatedContainers}}</td>
+ </tr>
+ <tr>
+ <td>Node Healthy Status</td>
+ <td>{{model.node.nodeHealthy}}</td>
+ </tr>
+ <tr>
+ <td>Last Node Health Report Time</td>
+ <td>{{model.node.lastNodeUpdateTime}}</td>
+ </tr>
+ <tr>
+ <td>Node Health Report</td>
+ <td>{{model.node.healthReport}}</td>
+ </tr>
+ {{#if model.node.nmStartupTime}}
+ <tr>
+ <td>Node Manager Start Time</td>
+ <td>{{model.node.nmStartupTime}}</td>
+ </tr>
+ {{/if}}
+ <tr>
+ <td>Node Manager Version</td>
+ <td>{{model.node.nodeManagerBuildVersion}}</td>
+ </tr>
+ <tr>
+ <td>Hadoop Version</td>
+ <td>{{model.node.hadoopBuildVersion}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+
+ <div class="row">
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - Memory
+ </div>
+ <div class="container-fluid" id="mem-donut-chart">
+ {{donut-chart data=model.rmNode.getMemoryDataForDonutChart
+ showLabels=true
+ parentId="mem-donut-chart"
+ ratio=0.6
+ type="memory"
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - VCores
+ </div>
+ <div class="container-fluid" id="vcore-donut-chart">
+ {{donut-chart data=model.rmNode.getVCoreDataForDonutChart
+ showLabels=true
+ parentId="vcore-donut-chart"
+ ratio=0.6
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+ </div>
+
+ {{#if model.nmGpuInfo}}
+ <div class="row">
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ <li>
+ Resources - yarn.io/gpu
+ </li>
+ </div>
+ <div class="container-fluid" id="gpu-donut-chart">
+ {{donut-chart data=model.rmNode.getGpuDataForDonutChart
+ showLabels=true
+ parentId="gpu-donut-chart"
+ ratio=0.6
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+ </div>
+ {{/if}}
+ </div>
+ </div>
+</div>
+{{outlet}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
new file mode 100644
index 0000000..55840ad
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-node/yarn-nm-gpu.hbs
@@ -0,0 +1,53 @@
+{{!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+--}}
+
+{{breadcrumb-bar breadcrumbs=breadcrumbs}}
+
+<div class="col-md-12 container-fluid">
+ <div class="row">
+
+ {{node-menu-panel path="yarn-node" nodeId=model.rmNode.id
+ nodeAddr=model.node.id nmGpuInfo=model.nmGpuInfo}}
+
+ <div class="col-md-10 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">Gpu Information</div>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Vendor</td>
+ <td>NVIDIA</td>
+ </tr>
+ <tr>
+ <td>Driver Version</td>
+ <td>{{model.nmGpuInfo.info.gpuDeviceInformation.driverVersion}}</td>
+ </tr>
+ <tr>
+ <td>Total Number Of Gpus</td>
+ <td>{{model.nmGpuInfo.info.totalGpuDevices.length}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+
+ {{#each model.nmGpuInfo.info.gpuDeviceInformation.gpus as |gpu|}}
+ {{yarn-nm-gpu-info gpu=gpu}}
+ {{/each}}
+ </div>
+ </div>
+</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
index b9da222..74cc916 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/converter.js
@@ -130,6 +130,57 @@ export default {
}
return value.toFixed(1) + " " + unit;
},
+ resourceToSimplifiedUnit: function (value, unit) {
+ // First convert unit to base unit ("").
+ var normalizedValue = value;
+ if (unit === "Ki") {
+ normalizedValue = normalizedValue * 1024;
+ } else if (unit === "Mi") {
+ normalizedValue = normalizedValue * 1024 * 1024;
+ } else if (unit === "Gi") {
+ normalizedValue = normalizedValue * 1024 * 1024 * 1024;
+ } else if (unit === "Ti") {
+ normalizedValue = normalizedValue * 1024 * 1024 * 1024 * 1024;
+ } else if (unit === "Pi") {
+ normalizedValue = normalizedValue * 1024 * 1024 * 1024 * 1024 * 1024;
+ } else if (unit === "K" || unit === "k") {
+ normalizedValue = normalizedValue * 1000;
+ } else if (unit === "M" || unit === "m") {
+ normalizedValue = normalizedValue * 1000 * 1000;
+ } else if (unit === "G" || unit === "g") {
+ normalizedValue = normalizedValue * 1000 * 1000 * 1000;
+ } else if (unit === "T" || unit === "t") {
+ normalizedValue = normalizedValue * 1000 * 1000 * 1000 * 1000;
+ } else if (unit === "P" || unit === "p") {
+ normalizedValue = normalizedValue * 1000 * 1000 * 1000 * 1000 * 1000;
+ }
+
+ // From baseunit ("") convert to most human readable unit
+ // (which value < 1024 * 0.9).
+ var finalUnit = "";
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Ki";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Mi";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Gi";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Ti";
+ }
+ if (normalizedValue / 1024 >= 0.9) {
+ normalizedValue = normalizedValue / 1024;
+ finalUnit = "Pi";
+ }
+
+ return normalizedValue.toFixed(1) + " " + finalUnit;
+ },
msToElapsedTimeUnit: function(millisecs, short) {
var seconds = Math.floor(millisecs / 1000);
var days = Math.floor(seconds / (3600 * 24));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/49] hadoop git commit: YARN-7254. UI and metrics changes related
to absolute resource configuration. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7254. UI and metrics changes related to absolute resource configuration. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c2440930
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c2440930
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c2440930
Branch: refs/heads/YARN-5881
Commit: c244093074d8bf9728ffe31778c3c621ed157c78
Parents: 6bf4a43
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 16 11:04:05 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../yarn/api/records/QueueConfigurations.java | 76 +++++++++++
.../yarn/api/records/ResourceInformation.java | 6 +
.../src/main/proto/yarn_protos.proto | 4 +
.../impl/pb/QueueConfigurationsPBImpl.java | 136 ++++++++++++++++++-
.../resource/DominantResourceCalculator.java | 24 +++-
.../scheduler/QueueResourceQuotas.java | 38 ------
.../scheduler/capacity/AbstractCSQueue.java | 18 ++-
.../scheduler/capacity/CSQueue.java | 4 +-
.../scheduler/capacity/LeafQueue.java | 4 +-
.../scheduler/capacity/ParentQueue.java | 56 ++++++--
.../scheduler/capacity/UsersManager.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 23 +++-
.../webapp/CapacitySchedulerPage.java | 49 +++++--
.../webapp/dao/CapacitySchedulerInfo.java | 3 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 6 +-
.../webapp/dao/CapacitySchedulerQueueInfo.java | 10 +-
.../dao/PartitionQueueCapacitiesInfo.java | 34 ++++-
.../webapp/dao/QueueCapacitiesInfo.java | 13 +-
...CapacitySchedulerWithMultiResourceTypes.java | 25 ++--
.../scheduler/capacity/TestLeafQueue.java | 24 +++-
20 files changed, 443 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
index e25c8aa..30096bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java
@@ -147,4 +147,80 @@ public abstract class QueueConfigurations {
@Private
@Unstable
public abstract void setMaxAMPercentage(float maxAMPercentage);
+
+ /**
+ * Get the effective minimum capacity of queue (from absolute resource).
+ *
+ * @return minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getEffectiveMinCapacity();
+
+ /**
+ * Set the effective minimum capacity of queue (from absolute resource).
+ *
+ * @param capacity
+ * minimum resource capability
+ */
+ @Private
+ @Unstable
+ public abstract void setEffectiveMinCapacity(Resource capacity);
+
+ /**
+ * Get the effective maximum capacity of queue (from absolute resource).
+ *
+ * @return maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getEffectiveMaxCapacity();
+
+ /**
+ * Set the effective maximum capacity of queue (from absolute resource).
+ *
+ * @param capacity
+ * maximum resource capability
+ */
+ @Private
+ @Unstable
+ public abstract void setEffectiveMaxCapacity(Resource capacity);
+
+ /**
+ * Get the configured minimum capacity of queue (from absolute resource).
+ *
+ * @return minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getConfiguredMinCapacity();
+
+ /**
+ * Set the configured minimum capacity of queue (from absolute resource).
+ *
+ * @param configuredMinResource
+ * minimum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract void setConfiguredMinCapacity(Resource configuredMinResource);
+
+ /**
+ * Get the configured maximum capacity of queue (from absolute resource).
+ *
+ * @return maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract Resource getConfiguredMaxCapacity();
+
+ /**
+ * Set the configured maximum capacity of queue (from absolute resource).
+ *
+ * @param configuredMaxResource
+ * maximum resource capability
+ */
+ @Public
+ @Unstable
+ public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 67592cc..a4c1f6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -225,6 +225,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
Long.MAX_VALUE);
}
+ public static ResourceInformation newInstance(String name, String units,
+ long minRes, long maxRes) {
+ return ResourceInformation.newInstance(name, units, 0L,
+ ResourceTypes.COUNTABLE, minRes, maxRes);
+ }
+
public static ResourceInformation newInstance(String name, long value) {
return ResourceInformation
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index fdbe2d4..3a9662b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -551,6 +551,10 @@ message QueueConfigurationsProto {
optional float maxCapacity = 3;
optional float absoluteMaxCapacity = 4;
optional float maxAMPercentage = 5;
+ optional ResourceProto effectiveMinCapacity = 6;
+ optional ResourceProto effectiveMaxCapacity = 7;
+ optional ResourceProto configuredMinCapacity = 8;
+ optional ResourceProto configuredMaxCapacity = 9;
}
message QueueConfigurationsMapProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index f308bce..80ef4b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -19,16 +19,22 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
import org.apache.hadoop.yarn.api.records.QueueConfigurations;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import com.google.protobuf.TextFormat;
public class QueueConfigurationsPBImpl extends QueueConfigurations {
- QueueConfigurationsProto proto =
- QueueConfigurationsProto.getDefaultInstance();
+ QueueConfigurationsProto proto = QueueConfigurationsProto
+ .getDefaultInstance();
QueueConfigurationsProto.Builder builder = null;
+ Resource configuredMinResource = null;
+ Resource configuredMaxResource = null;
+ Resource effMinResource = null;
+ Resource effMaxResource = null;
boolean viaProto = false;
public QueueConfigurationsPBImpl() {
@@ -41,11 +47,40 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
}
public QueueConfigurationsProto getProto() {
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
}
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.effMinResource != null) {
+ builder
+ .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource));
+ }
+ if (this.effMaxResource != null) {
+ builder
+ .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource));
+ }
+ if (this.configuredMinResource != null) {
+ builder.setEffectiveMinCapacity(
+ convertToProtoFormat(this.configuredMinResource));
+ }
+ if (this.configuredMaxResource != null) {
+ builder.setEffectiveMaxCapacity(
+ convertToProtoFormat(this.configuredMaxResource));
+ }
+ }
+
@Override
public float getCapacity() {
QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
@@ -106,6 +141,58 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
builder.setMaxAMPercentage(maxAMPercentage);
}
+ @Override
+ public Resource getEffectiveMinCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.effMinResource != null) {
+ return this.effMinResource;
+ }
+ if (!p.hasEffectiveMinCapacity()) {
+ return null;
+ }
+ this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity());
+ return this.effMinResource;
+ }
+
+ @Override
+ public void setEffectiveMinCapacity(Resource capacity) {
+ maybeInitBuilder();
+ if (capacity == null) {
+ builder.clearEffectiveMinCapacity();
+ }
+ this.effMinResource = capacity;
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.effMaxResource != null) {
+ return this.effMaxResource;
+ }
+ if (!p.hasEffectiveMaxCapacity()) {
+ return null;
+ }
+ this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity());
+ return this.effMaxResource;
+ }
+
+ @Override
+ public void setEffectiveMaxCapacity(Resource capacity) {
+ maybeInitBuilder();
+ if (capacity == null) {
+ builder.clearEffectiveMaxCapacity();
+ }
+ this.effMaxResource = capacity;
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
+ return new ResourcePBImpl(p);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource t) {
+ return ProtoUtils.convertToProtoFormat(t);
+ }
+
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = QueueConfigurationsProto.newBuilder(proto);
@@ -134,4 +221,49 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
return false;
}
+ @Override
+ public Resource getConfiguredMinCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.configuredMinResource != null) {
+ return this.configuredMinResource;
+ }
+ if (!p.hasConfiguredMinCapacity()) {
+ return null;
+ }
+ this.configuredMinResource = convertFromProtoFormat(
+ p.getConfiguredMinCapacity());
+ return this.configuredMinResource;
+ }
+
+ @Override
+ public void setConfiguredMinCapacity(Resource configuredMinResource) {
+ maybeInitBuilder();
+ if (configuredMinResource == null) {
+ builder.clearConfiguredMinCapacity();
+ }
+ this.configuredMinResource = configuredMinResource;
+ }
+
+ @Override
+ public Resource getConfiguredMaxCapacity() {
+ QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.configuredMaxResource != null) {
+ return this.configuredMaxResource;
+ }
+ if (!p.hasConfiguredMaxCapacity()) {
+ return null;
+ }
+ this.configuredMaxResource = convertFromProtoFormat(
+ p.getConfiguredMaxCapacity());
+ return this.configuredMaxResource;
+ }
+
+ @Override
+ public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+ maybeInitBuilder();
+ if (configuredMaxResource == null) {
+ builder.clearConfiguredMaxCapacity();
+ }
+ this.configuredMaxResource = configuredMaxResource;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 16e4527..cc4acf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -570,8 +570,26 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public Resource normalizeDown(Resource r, Resource stepFactor) {
- return Resources.createResource(
- roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
- roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+ Resource ret = Resource.newInstance(r);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation rResourceInformation = r.getResourceInformation(i);
+ ResourceInformation stepFactorResourceInformation = stepFactor
+ .getResourceInformation(i);
+ ResourceInformation tmp = ret.getResourceInformation(i);
+
+ long rValue = rResourceInformation.getValue();
+ long stepFactorValue = UnitsConversionUtil.convert(
+ stepFactorResourceInformation.getUnits(),
+ rResourceInformation.getUnits(),
+ stepFactorResourceInformation.getValue());
+
+ long value = rValue;
+ if (stepFactorValue != 0) {
+ value = roundDown(rValue, stepFactorValue);
+ }
+ tmp.setValue(value);
+ }
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
index 2e653fc..08b4d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -112,42 +112,4 @@ public class QueueResourceQuotas extends AbstractResourceUsage {
public void setEffectiveMaxResource(String label, Resource res) {
_set(label, ResourceType.EFF_MAX_RESOURCE, res);
}
-
- /*
- * Effective Minimum Resource
- */
- public Resource getEffectiveMinResourceUp() {
- return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
- }
-
- public Resource getEffectiveMinResourceUp(String label) {
- return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
- }
-
- public void setEffectiveMinResourceUp(String label, Resource res) {
- _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
- }
-
- public void setEffectiveMinResourceUp(Resource res) {
- _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
- }
-
- /*
- * Effective Maximum Resource
- */
- public Resource getEffectiveMaxResourceUp() {
- return getEffectiveMaxResourceUp(NL);
- }
-
- public Resource getEffectiveMaxResourceUp(String label) {
- return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
- }
-
- public void setEffectiveMaxResourceUp(Resource res) {
- setEffectiveMaxResourceUp(NL, res);
- }
-
- public void setEffectiveMaxResourceUp(String label, Resource res) {
- _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index dddac4d..9caf589 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -398,6 +398,10 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maxResource = conf.getMaximumResourceRequirement(label,
queuePath, resourceTypes);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("capacityConfigType is '" + capacityConfigType
+ + "' for queue '" + getQueueName());
+ }
if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
this.capacityConfigType = (!minResource.equals(Resources.none())
&& queueCapacities.getAbsoluteCapacity(label) == 0f)
@@ -481,12 +485,6 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public Resource getEffectiveCapacityUp(String label) {
- return Resources
- .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
- }
-
- @Override
public Resource getEffectiveCapacityDown(String label, Resource factor) {
return Resources.normalizeDown(resourceCalculator,
getQueueResourceQuotas().getEffectiveMinResource(label),
@@ -621,6 +619,14 @@ public abstract class AbstractCSQueue implements CSQueue {
queueConfiguration.setMaxCapacity(maxCapacity);
queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity);
queueConfiguration.setMaxAMPercentage(maxAMPercentage);
+ queueConfiguration.setConfiguredMinCapacity(
+ queueResourceQuotas.getConfiguredMinResource(nodeLabel));
+ queueConfiguration.setConfiguredMaxCapacity(
+ queueResourceQuotas.getConfiguredMaxResource(nodeLabel));
+ queueConfiguration.setEffectiveMinCapacity(
+ queueResourceQuotas.getEffectiveMinResource(nodeLabel));
+ queueConfiguration.setEffectiveMaxCapacity(
+ queueResourceQuotas.getEffectiveMaxResource(nodeLabel));
queueConfigurations.put(nodeLabel, queueConfiguration);
}
return queueConfigurations;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 2e29a71..624fcc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -25,7 +25,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.security.AccessControlException;
@@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* <code>CSQueue</code> represents a node in the tree of
* hierarchical queues in the {@link CapacityScheduler}.
@@ -380,7 +381,6 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* @return effective queue capacity
*/
Resource getEffectiveCapacity(String label);
- Resource getEffectiveCapacityUp(String label);
Resource getEffectiveCapacityDown(String label, Resource factor);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1460121..41ec4ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,7 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
- Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+ Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
Resource userAMLimit = Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
@@ -700,7 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
* non-labeled), * with per-partition am-resource-percent to get the max am
* resource limit for this queue and partition.
*/
- Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
+ Resource queuePartitionResource = getEffectiveCapacity(nodePartition);
Resource queueCurrentLimit = Resources.none();
// For non-labeled partition, we need to consider the current queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index c770fac..5ab1494 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -914,9 +914,11 @@ public class ParentQueue extends AbstractCSQueue {
queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ queueCapacities.setAbsoluteCapacity(label, 1.0f);
}
- // Total configured min resources of direct children of queue
+ // Total configured min resources of direct children of this given parent
+ // queue.
Resource configuredMinResources = Resource.newInstance(0L, 0);
for (CSQueue childQueue : getChildQueues()) {
Resources.addTo(configuredMinResources,
@@ -960,7 +962,7 @@ public class ParentQueue extends AbstractCSQueue {
// present could also be taken from effective max resource of parent.
Resource parentMaxRes = queueResourceQuotas
.getConfiguredMaxResource(label);
- if (parentMaxRes.equals(Resources.none())) {
+ if (parent != null && parentMaxRes.equals(Resources.none())) {
parentMaxRes = parent.getQueueResourceQuotas()
.getEffectiveMaxResource(label);
}
@@ -976,6 +978,11 @@ public class ParentQueue extends AbstractCSQueue {
parentMaxRes);
childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
Resources.clone(effMaxResource));
+
+ // In cases where we still need to update some units based on
+ // percentage, we have to calculate percentage and update.
+ deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc,
+ childQueue);
} else {
childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
Resources.multiply(resourceByLabel,
@@ -983,16 +990,6 @@ public class ParentQueue extends AbstractCSQueue {
childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
.getAbsoluteMaximumCapacity(label)));
-
- childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
- Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
- childQueue.getQueueCapacities().getAbsoluteCapacity(label),
- minimumAllocation));
- childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
- Resources.multiplyAndNormalizeUp(rc,
- resourceByLabel, childQueue.getQueueCapacities()
- .getAbsoluteMaximumCapacity(label),
- minimumAllocation));
}
if (LOG.isDebugEnabled()) {
@@ -1006,6 +1003,41 @@ public class ParentQueue extends AbstractCSQueue {
}
}
+ private void deriveCapacityFromAbsoluteConfigurations(String label,
+ Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
+
+ /*
+ * In case when queues are configured with absolute resources, it is better
+ * to update capacity/max-capacity etc w.r.t absolute resource as well. In
+ * case of computation, these values wont be used any more. However for
+ * metrics and UI, its better these values are pre-computed here itself.
+ */
+
+ // 1. Update capacity as a float based on parent's minResource
+ childQueue.getQueueCapacities().setCapacity(label,
+ rc.divide(clusterResource,
+ childQueue.getQueueResourceQuotas().getEffectiveMinResource(label),
+ getQueueResourceQuotas().getEffectiveMinResource(label)));
+
+ // 2. Update max-capacity as a float based on parent's maxResource
+ childQueue.getQueueCapacities().setMaximumCapacity(label,
+ rc.divide(clusterResource,
+ childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label),
+ getQueueResourceQuotas().getEffectiveMaxResource(label)));
+
+ // 3. Update absolute capacity as a float based on parent's minResource and
+ // cluster resource.
+ childQueue.getQueueCapacities().setAbsoluteCapacity(label,
+ (float) childQueue.getQueueCapacities().getCapacity()
+ / getQueueCapacities().getAbsoluteCapacity(label));
+
+ // 4. Update absolute max-capacity as a float based on parent's maxResource
+ // and cluster resource.
+ childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
+ (float) childQueue.getQueueCapacities().getMaximumCapacity(label)
+ / getQueueCapacities().getAbsoluteMaximumCapacity(label));
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index efc20e9..7287c5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,7 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
+ Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index e9bee14..e3e47b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
@@ -905,6 +906,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
StringBuilder diagnosticMessage) {
LeafQueue queue = getCSLeafQueue();
QueueCapacities queueCapacities = queue.getQueueCapacities();
+ QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas();
diagnosticMessage.append(" Details : AM Partition = ");
diagnosticMessage.append(appAMNodePartitionName.isEmpty()
? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
@@ -926,6 +928,18 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
* 100);
diagnosticMessage.append(" % ; ");
+ diagnosticMessage.append("Queue's capacity (absolute resource) = ");
+ diagnosticMessage.append(
+ queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's used capacity (absolute resource) = ");
+ diagnosticMessage
+ .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's max capacity (absolute resource) = ");
+ diagnosticMessage.append(
+ queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName));
+ diagnosticMessage.append(" ; ");
}
/**
@@ -989,13 +1003,10 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
if (!calc.isInvalidDivisor(totalPartitionRes)) {
- float queueAbsMaxCapPerPartition =
- ((AbstractCSQueue) getQueue()).getQueueCapacities()
- .getAbsoluteCapacity(getAppAMNodePartitionName());
+ Resource effCap = ((AbstractCSQueue) getQueue())
+ .getEffectiveCapacity(getAppAMNodePartitionName());
float queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(),
- Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition))
- * 100;
+ report.getUsedResources(), effCap) * 100;
report.setQueueUsagePercentage(queueUsagePerc);
}
return report;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 74b4e79..3f68e07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -155,21 +155,36 @@ class CapacitySchedulerPage extends RmView {
? new ResourceInfo(Resources.none())
: resourceUsages.getAmUsed();
ri.
- __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)).
- __("Configured Capacity:", percent(capacities.getCapacity() / 100)).
- __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)).
- __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)).
- __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)).
- __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)).
- __("Used Resources:", resourceUsages.getUsed().toString()).
- __("Configured Max Application Master Limit:", StringUtils.format("%.1f",
- capacities.getMaxAMLimitPercentage())).
- __("Max Application Master Resources:",
- resourceUsages.getAMLimit().toString()).
- __("Used Application Master Resources:",
- amUsed.toString()).
- __("Max Application Master Resources Per User:",
- userAMResourceLimit.toString());
+ __("Used Capacity:",
+ appendPercent(resourceUsages.getUsed().toString(),
+ capacities.getUsedCapacity() / 100))
+ .__("Configured Capacity:",
+ capacities.getConfiguredMinResource().toString())
+ .__("Configured Max Capacity:",
+ capacities.getConfiguredMaxResource().getResource()
+ .equals(Resources.none())
+ ? "unlimited"
+ : capacities.getConfiguredMaxResource().toString())
+ .__("Effective Capacity:",
+ appendPercent(capacities.getEffectiveMinResource().toString(),
+ capacities.getCapacity() / 100))
+ .__("Effective Max Capacity:",
+ appendPercent(capacities.getEffectiveMaxResource().toString(),
+ capacities.getMaxCapacity() / 100))
+ .__("Absolute Used Capacity:",
+ percent(capacities.getAbsoluteUsedCapacity() / 100))
+ .__("Absolute Configured Capacity:",
+ percent(capacities.getAbsoluteCapacity() / 100))
+ .__("Absolute Configured Max Capacity:",
+ percent(capacities.getAbsoluteMaxCapacity() / 100))
+ .__("Used Resources:", resourceUsages.getUsed().toString())
+ .__("Configured Max Application Master Limit:",
+ StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage()))
+ .__("Max Application Master Resources:",
+ resourceUsages.getAMLimit().toString())
+ .__("Used Application Master Resources:", amUsed.toString())
+ .__("Max Application Master Resources Per User:",
+ userAMResourceLimit.toString());
}
private void renderCommonLeafQueueInfo(ResponseInfo ri) {
@@ -615,6 +630,10 @@ class CapacitySchedulerPage extends RmView {
return QueuesBlock.class;
}
+ static String appendPercent(String message, float f) {
+ return message + " (" + StringUtils.formatPercent(f, 1) + ")";
+ }
+
static String percent(float f) {
return StringUtils.formatPercent(f, 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
index 32e4ac5..ca42041 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
@@ -59,7 +59,8 @@ public class CapacitySchedulerInfo extends SchedulerInfo {
max = 1f;
this.maxCapacity = max * 100;
- capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false);
+ capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(),
+ parent.getQueueResourceQuotas(), false);
queues = getQueues(parent);
health = new CapacitySchedulerHealthInfo(cs);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index 7dcdf58..343c78e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
@@ -89,8 +90,9 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
}
@Override
- protected void populateQueueCapacities(QueueCapacities qCapacities) {
- capacities = new QueueCapacitiesInfo(qCapacities);
+ protected void populateQueueCapacities(QueueCapacities qCapacities,
+ QueueResourceQuotas qResQuotas) {
+ capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas);
}
public int getNumActiveApplications() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 86b2fea..d4de9ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue;
@@ -103,7 +104,8 @@ public class CapacitySchedulerQueueInfo {
Collections.sort(nodeLabels);
}
QueueCapacities qCapacities = q.getQueueCapacities();
- populateQueueCapacities(qCapacities);
+ QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas();
+ populateQueueCapacities(qCapacities, qResQuotas);
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
@@ -118,8 +120,10 @@ public class CapacitySchedulerQueueInfo {
resources = new ResourcesInfo(queueResourceUsage, false);
}
- protected void populateQueueCapacities(QueueCapacities qCapacities) {
- capacities = new QueueCapacitiesInfo(qCapacities, false);
+ protected void populateQueueCapacities(QueueCapacities qCapacities,
+ QueueResourceQuotas qResQuotas) {
+ capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas,
+ false);
}
public float getCapacity() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
index 5e298f9..2a15502 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java
@@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
/**
* This class represents queue capacities for a given partition
*/
@@ -36,13 +39,19 @@ public class PartitionQueueCapacitiesInfo {
private float absoluteUsedCapacity;
private float absoluteMaxCapacity = 100;
private float maxAMLimitPercentage;
+ private ResourceInfo configuredMinResource;
+ private ResourceInfo configuredMaxResource;
+ private ResourceInfo effectiveMinResource;
+ private ResourceInfo effectiveMaxResource;
public PartitionQueueCapacitiesInfo() {
}
public PartitionQueueCapacitiesInfo(String partitionName, float capacity,
float usedCapacity, float maxCapacity, float absCapacity,
- float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) {
+ float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage,
+ Resource confMinRes, Resource confMaxRes, Resource effMinRes,
+ Resource effMaxRes) {
super();
this.partitionName = partitionName;
this.capacity = capacity;
@@ -52,6 +61,10 @@ public class PartitionQueueCapacitiesInfo {
this.absoluteUsedCapacity = absUsedCapacity;
this.absoluteMaxCapacity = absMaxCapacity;
this.maxAMLimitPercentage = maxAMLimitPercentage;
+ this.configuredMinResource = new ResourceInfo(confMinRes);
+ this.configuredMaxResource = new ResourceInfo(confMaxRes);
+ this.effectiveMinResource = new ResourceInfo(effMinRes);
+ this.effectiveMaxResource = new ResourceInfo(effMaxRes);
}
public float getCapacity() {
@@ -117,4 +130,23 @@ public class PartitionQueueCapacitiesInfo {
public void setMaxAMLimitPercentage(float maxAMLimitPercentage) {
this.maxAMLimitPercentage = maxAMLimitPercentage;
}
+
+ public ResourceInfo getConfiguredMinResource() {
+ return configuredMinResource;
+ }
+
+ public ResourceInfo getConfiguredMaxResource() {
+ if (configuredMaxResource.getResource().equals(Resources.none())) {
+ return null;
+ }
+ return configuredMaxResource;
+ }
+
+ public ResourceInfo getEffectiveMinResource() {
+ return effectiveMinResource;
+ }
+
+ public ResourceInfo getEffectiveMaxResource() {
+ return effectiveMaxResource;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
index 9a3e439..35c80d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java
@@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
/**
@@ -39,6 +40,7 @@ public class QueueCapacitiesInfo {
}
public QueueCapacitiesInfo(QueueCapacities capacities,
+ QueueResourceQuotas resourceQuotas,
boolean considerAMUsage) {
if (capacities == null) {
return;
@@ -68,12 +70,17 @@ public class QueueCapacitiesInfo {
queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo(
partitionName, capacity, usedCapacity, maxCapacity, absCapacity,
absUsedCapacity, absMaxCapacity,
- considerAMUsage ? maxAMLimitPercentage : 0f));
+ considerAMUsage ? maxAMLimitPercentage : 0f,
+ resourceQuotas.getConfiguredMinResource(partitionName),
+ resourceQuotas.getConfiguredMaxResource(partitionName),
+ resourceQuotas.getEffectiveMinResource(partitionName),
+ resourceQuotas.getEffectiveMaxResource(partitionName)));
}
}
- public QueueCapacitiesInfo(QueueCapacities capacities) {
- this(capacities, true);
+ public QueueCapacitiesInfo(QueueCapacities capacities,
+ QueueResourceQuotas resourceQuotas) {
+ this(capacities, resourceQuotas, true);
}
public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
index b386c18..38768e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java
@@ -61,11 +61,20 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
Map<String, ResourceInformation> riMap = new HashMap<>();
// Initialize mandatory resources
- riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB);
- riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES);
- riMap.put(RESOURCE_1, ResourceInformation
- .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0,
- Integer.MAX_VALUE));
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+ ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
@@ -107,7 +116,7 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
- Assert.assertEquals(Resource.newInstance(1 * GB, 0),
+ Assert.assertEquals(Resource.newInstance(1 * GB, 1),
leafQueue.getUsedResources());
RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
@@ -123,9 +132,9 @@ public class TestCapacitySchedulerWithMultiResourceTypes {
// Do node heartbeats 1 time and check container allocated.
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
- // Now used resource = <mem=1GB, vcore=0> + <mem=2GB,vcore=2,res_1=2>
+ // Now used resource = <mem=1GB, vcore=1> + <mem=2GB,vcore=2,res_1=2>
Assert.assertEquals(
- TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)),
+ TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)),
leafQueue.getUsedResources());
// Acquire container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c2440930/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 96f8970..30bff78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -1355,7 +1356,7 @@ public class TestLeafQueue {
// TODO, fix headroom in the future patch
assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
// User limit = 2G, 2 in use
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+ assertEquals(1*GB, app_1.getHeadroom().getMemorySize());
// the application is not yet active
// Again one to user_0 since he hasn't exceeded user limit yet
@@ -1366,8 +1367,8 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemorySize());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
- assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
+ assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
@@ -3874,7 +3875,7 @@ public class TestLeafQueue {
// Queue "test" consumes 100% of the cluster, so its capacity and absolute
// capacity are both 1.0f.
- Queue queue = createQueue("test", null, 1.0f, 1.0f);
+ Queue queue = createQueue("test", null, 1.0f, 1.0f, res);
final String user = "user1";
FiCaSchedulerApp app =
new FiCaSchedulerApp(appAttId, user, queue,
@@ -3891,7 +3892,8 @@ public class TestLeafQueue {
// Queue "test2" is a child of root and its capacity is 50% of root. As a
// child of root, its absolute capaicty is also 50%.
- queue = createQueue("test2", null, 0.5f, 0.5f);
+ queue = createQueue("test2", null, 0.5f, 0.5f,
+ Resources.divideAndCeil(dominantResourceCalculator, res, 2));
app = new FiCaSchedulerApp(appAttId, user, queue,
queue.getAbstractUsersManager(), rmContext);
app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3903,7 +3905,8 @@ public class TestLeafQueue {
// Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
// Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
- AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+ AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f,
+ Resources.divideAndCeil(dominantResourceCalculator, res, 4));
app = new FiCaSchedulerApp(appAttId, user, qChild,
qChild.getAbstractUsersManager(), rmContext);
app.getAppAttemptResourceUsage().incUsed(requestedResource);
@@ -3922,7 +3925,7 @@ public class TestLeafQueue {
}
private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
- float absCap) {
+ float absCap, Resource res) {
CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
null, QueueState.RUNNING, null, "", null, false);
@@ -3934,6 +3937,13 @@ public class TestLeafQueue {
QueueCapacities qCaps = mock(QueueCapacities.class);
when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
when(queue.getQueueCapacities()).thenReturn(qCaps);
+ QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class);
+ when(qQuotas.getConfiguredMinResource(any())).thenReturn(res);
+ when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res);
+ when(qQuotas.getEffectiveMinResource(any())).thenReturn(res);
+ when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res);
+ when(queue.getQueueResourceQuotas()).thenReturn(qQuotas);
+ when(queue.getEffectiveCapacity(any())).thenReturn(res);
return queue;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
new file mode 100644
index 0000000..4d7f524
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitProtocol.java
@@ -0,0 +1,1371 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.MapFile;
+import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.concurrent.HadoopExecutors;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.test.LambdaTestUtils.*;
+
+/**
+ * Test the job/task commit actions of an S3A Committer, including trying to
+ * simulate some failure and retry conditions.
+ * Derived from
+ * {@code org.apache.hadoop.mapreduce.lib.output.TestFileOutputCommitter}.
+ *
+ * This is a complex test suite as it tries to explore the full lifecycle
+ * of committers, and is designed for subclassing.
+ */
+@SuppressWarnings({"unchecked", "ThrowableNotThrown", "unused"})
+public abstract class AbstractITCommitProtocol extends AbstractCommitITest {
+ private Path outDir;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractITCommitProtocol.class);
+
+ private static final String SUB_DIR = "SUB_DIR";
+
+ protected static final String PART_00000 = "part-m-00000";
+
+ /**
+ * Counter to guarantee that even in parallel test runs, no job has the same
+ * ID.
+ */
+
+ private String jobId;
+
+ // A random task attempt id for testing.
+ private String attempt0;
+ private TaskAttemptID taskAttempt0;
+
+ private String attempt1;
+ private TaskAttemptID taskAttempt1;
+
+ private static final Text KEY_1 = new Text("key1");
+ private static final Text KEY_2 = new Text("key2");
+ private static final Text VAL_1 = new Text("val1");
+ private static final Text VAL_2 = new Text("val2");
+
+ /** A job to abort in test case teardown. */
+ private List<JobData> abortInTeardown = new ArrayList<>(1);
+
+ private final StandardCommitterFactory
+ standardCommitterFactory = new StandardCommitterFactory();
+
+ private void cleanupDestDir() throws IOException {
+ rmdir(this.outDir, getConfiguration());
+ }
+
+ /**
+ * This must return the name of a suite which is unique to the non-abstract
+ * test.
+ * @return a string which must be unique and a valid path.
+ */
+ protected abstract String suitename();
+
+ /**
+ * Get the log; can be overridden for test case log.
+ * @return a log.
+ */
+ public Logger log() {
+ return LOG;
+ }
+
+ /**
+ * Overridden method returns the suitename as well as the method name,
+ * so if more than one committer test is run in parallel, paths are
+ * isolated.
+ * @return a name for a method, unique across the suites and test cases.
+ */
+ @Override
+ protected String getMethodName() {
+ return suitename() + "-" + super.getMethodName();
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ jobId = randomJobId();
+ attempt0 = "attempt_" + jobId + "_m_000000_0";
+ taskAttempt0 = TaskAttemptID.forName(attempt0);
+ attempt1 = "attempt_" + jobId + "_m_000001_0";
+ taskAttempt1 = TaskAttemptID.forName(attempt1);
+
+ outDir = path(getMethodName());
+ S3AFileSystem fileSystem = getFileSystem();
+ bindFileSystem(fileSystem, outDir, fileSystem.getConf());
+ abortMultipartUploadsUnderPath(outDir);
+ cleanupDestDir();
+ }
+
+ /**
+ * Create a random Job ID using the fork ID as part of the number.
+ * @return fork ID string in a format parseable by Jobs
+ * @throws Exception failure
+ */
+ private String randomJobId() throws Exception {
+ String testUniqueForkId = System.getProperty(TEST_UNIQUE_FORK_ID, "0001");
+ int l = testUniqueForkId.length();
+ String trailingDigits = testUniqueForkId.substring(l - 4, l);
+ try {
+ int digitValue = Integer.valueOf(trailingDigits);
+ return String.format("20070712%04d_%04d",
+ (long)(Math.random() * 1000),
+ digitValue);
+ } catch (NumberFormatException e) {
+ throw new Exception("Failed to parse " + trailingDigits, e);
+ }
+ }
+
+ @Override
+ public void teardown() throws Exception {
+ describe("teardown");
+ abortInTeardown.forEach(this::abortJobQuietly);
+ if (outDir != null) {
+ try {
+ abortMultipartUploadsUnderPath(outDir);
+ cleanupDestDir();
+ } catch (IOException e) {
+ log().info("Exception during cleanup", e);
+ }
+ }
+ S3AFileSystem fileSystem = getFileSystem();
+ if (fileSystem != null) {
+ log().info("Statistics for {}:\n{}", fileSystem.getUri(),
+ fileSystem.getInstrumentation().dump(" ", " = ", "\n", true));
+ }
+
+ super.teardown();
+ }
+
+ /**
+ * Add the specified job to the current list of jobs to abort in teardown.
+ * @param jobData job data.
+ */
+ protected void abortInTeardown(JobData jobData) {
+ abortInTeardown.add(jobData);
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ disableFilesystemCaching(conf);
+ bindCommitter(conf);
+ return conf;
+ }
+
+ /**
+ * Bind a path to the FS in the cache.
+ * @param fs filesystem
+ * @param path s3 path
+ * @param conf configuration
+ * @throws IOException any problem
+ */
+ private void bindFileSystem(FileSystem fs, Path path, Configuration conf)
+ throws IOException {
+ FileSystemTestHelper.addFileSystemForTesting(path.toUri(), conf, fs);
+ }
+
+ /***
+ * Bind to the committer from the methods of
+ * {@link #getCommitterFactoryName()} and {@link #getCommitterName()}.
+ * @param conf configuration to set up
+ */
+ protected void bindCommitter(Configuration conf) {
+ super.bindCommitter(conf, getCommitterFactoryName(), getCommitterName());
+ }
+
+ /**
+ * Create a committer for a task.
+ * @param context task context
+ * @return new committer
+ * @throws IOException failure
+ */
+ protected AbstractS3ACommitter createCommitter(
+ TaskAttemptContext context) throws IOException {
+ return createCommitter(getOutDir(), context);
+ }
+
+ /**
+ * Create a committer for a task and a given output path.
+ * @param outputPath path
+ * @param context task context
+ * @return new committer
+ * @throws IOException failure
+ */
+ protected abstract AbstractS3ACommitter createCommitter(
+ Path outputPath,
+ TaskAttemptContext context) throws IOException;
+
+
+ protected String getCommitterFactoryName() {
+ return CommitConstants.S3A_COMMITTER_FACTORY;
+ }
+
+ protected abstract String getCommitterName();
+
+ protected Path getOutDir() {
+ return outDir;
+ }
+
+ protected String getJobId() {
+ return jobId;
+ }
+
+ protected String getAttempt0() {
+ return attempt0;
+ }
+
+ protected TaskAttemptID getTaskAttempt0() {
+ return taskAttempt0;
+ }
+
+ protected String getAttempt1() {
+ return attempt1;
+ }
+
+ protected TaskAttemptID getTaskAttempt1() {
+ return taskAttempt1;
+ }
+
+ /**
+ * Functional interface for creating committers, designed to allow
+ * different factories to be used to create different failure modes.
+ */
+ @FunctionalInterface
+ public interface CommitterFactory {
+
+ /**
+ * Create a committer for a task.
+ * @param context task context
+ * @return new committer
+ * @throws IOException failure
+ */
+ AbstractS3ACommitter createCommitter(
+ TaskAttemptContext context) throws IOException;
+ }
+
+ /**
+ * The normal committer creation factory, uses the abstract methods
+ * in the class.
+ */
+ public class StandardCommitterFactory implements CommitterFactory {
+ @Override
+ public AbstractS3ACommitter createCommitter(TaskAttemptContext context)
+ throws IOException {
+ return AbstractITCommitProtocol.this.createCommitter(context);
+ }
+ }
+
+ /**
+ * Write some text out.
+ * @param context task
+ * @throws IOException IO failure
+ * @throws InterruptedException write interrupted
+ */
+ protected void writeTextOutput(TaskAttemptContext context)
+ throws IOException, InterruptedException {
+ describe("write output");
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Writing Text output for task %s", context.getTaskAttemptID())) {
+ writeOutput(new LoggingTextOutputFormat().getRecordWriter(context),
+ context);
+ }
+ }
+
+ /**
+ * Write the standard output.
+ * @param writer record writer
+ * @param context task context
+ * @throws IOException IO failure
+ * @throws InterruptedException write interrupted
+ */
+ private void writeOutput(RecordWriter writer,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ NullWritable nullWritable = NullWritable.get();
+ try(CloseWriter cw = new CloseWriter(writer, context)) {
+ writer.write(KEY_1, VAL_1);
+ writer.write(null, nullWritable);
+ writer.write(null, VAL_1);
+ writer.write(nullWritable, VAL_2);
+ writer.write(KEY_2, nullWritable);
+ writer.write(KEY_1, null);
+ writer.write(null, null);
+ writer.write(KEY_2, VAL_2);
+ writer.close(context);
+ }
+ }
+
+ /**
+ * Write the output of a map.
+ * @param writer record writer
+ * @param context task context
+ * @throws IOException IO failure
+ * @throws InterruptedException write interrupted
+ */
+ private void writeMapFileOutput(RecordWriter writer,
+ TaskAttemptContext context) throws IOException, InterruptedException {
+ describe("\nWrite map output");
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Writing Text output for task %s", context.getTaskAttemptID());
+ CloseWriter cw = new CloseWriter(writer, context)) {
+ for (int i = 0; i < 10; ++i) {
+ Text val = ((i & 1) == 1) ? VAL_1 : VAL_2;
+ writer.write(new LongWritable(i), val);
+ }
+ writer.close(context);
+ }
+ }
+
+ /**
+ * Details on a job for use in {@code startJob} and elsewhere.
+ */
+ public static class JobData {
+ private final Job job;
+ private final JobContext jContext;
+ private final TaskAttemptContext tContext;
+ private final AbstractS3ACommitter committer;
+ private final Configuration conf;
+
+ public JobData(Job job,
+ JobContext jContext,
+ TaskAttemptContext tContext,
+ AbstractS3ACommitter committer) {
+ this.job = job;
+ this.jContext = jContext;
+ this.tContext = tContext;
+ this.committer = committer;
+ conf = job.getConfiguration();
+ }
+ }
+
+ /**
+ * Create a new job. Sets the task attempt ID,
+ * and output dir; asks for a success marker.
+ * @return the new job
+ * @throws IOException failure
+ */
+ public Job newJob() throws IOException {
+ return newJob(outDir, getConfiguration(), attempt0);
+ }
+
+ /**
+ * Create a new job. Sets the task attempt ID,
+ * and output dir; asks for a success marker.
+ * @param dir dest dir
+ * @param configuration config to get the job from
+ * @param taskAttemptId task attempt
+ * @return the new job
+ * @throws IOException failure
+ */
+ private Job newJob(Path dir, Configuration configuration,
+ String taskAttemptId) throws IOException {
+ Job job = Job.getInstance(configuration);
+ Configuration conf = job.getConfiguration();
+ conf.set(MRJobConfig.TASK_ATTEMPT_ID, taskAttemptId);
+ conf.setBoolean(CREATE_SUCCESSFUL_JOB_OUTPUT_DIR_MARKER, true);
+ FileOutputFormat.setOutputPath(job, dir);
+ return job;
+ }
+
+ /**
+ * Start a job with a committer; optionally write the test data.
+ * Always register the job to be aborted (quietly) in teardown.
+ * This is, from an "OO-purity perspective" the wrong kind of method to
+ * do: it's setting things up, mixing functionality, registering for teardown.
+ * Its aim is simple though: a common body of code for starting work
+ * in test cases.
+ * @param writeText should the text be written?
+ * @return the job data 4-tuple
+ * @throws IOException IO problems
+ * @throws InterruptedException interruption during write
+ */
+ protected JobData startJob(boolean writeText)
+ throws IOException, InterruptedException {
+ return startJob(standardCommitterFactory, writeText);
+ }
+
+ /**
+ * Start a job with a committer; optionally write the test data.
+ * Always register the job to be aborted (quietly) in teardown.
+ * This is, from an "OO-purity perspective" the wrong kind of method to
+ * do: it's setting things up, mixing functionality, registering for teardown.
+ * Its aim is simple though: a common body of code for starting work
+ * in test cases.
+ * @param factory the committer factory to use
+ * @param writeText should the text be written?
+ * @return the job data 4-tuple
+ * @throws IOException IO problems
+ * @throws InterruptedException interruption during write
+ */
+ protected JobData startJob(CommitterFactory factory, boolean writeText)
+ throws IOException, InterruptedException {
+ Job job = newJob();
+ Configuration conf = job.getConfiguration();
+ conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt0);
+ conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
+ JobContext jContext = new JobContextImpl(conf, taskAttempt0.getJobID());
+ TaskAttemptContext tContext = new TaskAttemptContextImpl(conf,
+ taskAttempt0);
+ AbstractS3ACommitter committer = factory.createCommitter(tContext);
+
+ // setup
+ JobData jobData = new JobData(job, jContext, tContext, committer);
+ setup(jobData);
+ abortInTeardown(jobData);
+
+ if (writeText) {
+ // write output
+ writeTextOutput(tContext);
+ }
+ return jobData;
+ }
+
+ /**
+ * Set up the job and task.
+ * @param jobData job data
+ * @throws IOException problems
+ */
+ protected void setup(JobData jobData) throws IOException {
+ AbstractS3ACommitter committer = jobData.committer;
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+ describe("\nsetup job");
+ try (DurationInfo d = new DurationInfo(LOG,
+ "setup job %s", jContext.getJobID())) {
+ committer.setupJob(jContext);
+ }
+ try (DurationInfo d = new DurationInfo(LOG,
+ "setup task %s", tContext.getTaskAttemptID())) {
+ committer.setupTask(tContext);
+ }
+ describe("setup complete\n");
+ }
+
+ /**
+ * Abort a job quietly.
+ * @param jobData job info
+ */
+ protected void abortJobQuietly(JobData jobData) {
+ abortJobQuietly(jobData.committer, jobData.jContext, jobData.tContext);
+ }
+
+ /**
+ * Abort a job quietly: first task, then job.
+ * @param committer committer
+ * @param jContext job context
+ * @param tContext task context
+ */
+ protected void abortJobQuietly(AbstractS3ACommitter committer,
+ JobContext jContext,
+ TaskAttemptContext tContext) {
+ describe("\naborting task");
+ try {
+ committer.abortTask(tContext);
+ } catch (IOException e) {
+ log().warn("Exception aborting task:", e);
+ }
+ describe("\naborting job");
+ try {
+ committer.abortJob(jContext, JobStatus.State.KILLED);
+ } catch (IOException e) {
+ log().warn("Exception aborting job", e);
+ }
+ }
+
+ /**
+ * Commit up the task and then the job.
+ * @param committer committer
+ * @param jContext job context
+ * @param tContext task context
+ * @throws IOException problems
+ */
+ protected void commit(AbstractS3ACommitter committer,
+ JobContext jContext,
+ TaskAttemptContext tContext) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "committing work", jContext.getJobID())) {
+ describe("\ncommitting task");
+ committer.commitTask(tContext);
+ describe("\ncommitting job");
+ committer.commitJob(jContext);
+ describe("commit complete\n");
+ }
+ }
+
+ /**
+ * Execute work as part of a test, after creating the job.
+ * After the execution, {@link #abortJobQuietly(JobData)} is
+ * called for abort/cleanup.
+ * @param name name of work (for logging)
+ * @param action action to execute
+ * @throws Exception failure
+ */
+ protected void executeWork(String name, ActionToTest action)
+ throws Exception {
+ executeWork(name, startJob(false), action);
+ }
+
+ /**
+ * Execute work as part of a test, against the created job.
+ * After the execution, {@link #abortJobQuietly(JobData)} is
+ * called for abort/cleanup.
+ * @param name name of work (for logging)
+ * @param jobData job info
+ * @param action action to execute
+ * @throws Exception failure
+ */
+ public void executeWork(String name,
+ JobData jobData,
+ ActionToTest action) throws Exception {
+ try (DurationInfo d = new DurationInfo(LOG, "Executing %s", name)) {
+ action.exec(jobData.job,
+ jobData.jContext,
+ jobData.tContext,
+ jobData.committer);
+ } finally {
+ abortJobQuietly(jobData);
+ }
+ }
+
+ /**
+ * Verify that recovery doesn't work for these committers.
+ */
+ @Test
+ @SuppressWarnings("deprecation")
+ public void testRecoveryAndCleanup() throws Exception {
+ describe("Test (unsupported) task recovery.");
+ JobData jobData = startJob(true);
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+
+ assertNotNull("null workPath in committer " + committer,
+ committer.getWorkPath());
+ assertNotNull("null outputPath in committer " + committer,
+ committer.getOutputPath());
+
+ // Commit the task. This will promote data and metadata to where
+ // job commits will pick it up on commit or abort.
+ committer.commitTask(tContext);
+ assertTaskAttemptPathDoesNotExist(committer, tContext);
+
+ Configuration conf2 = jobData.job.getConfiguration();
+ conf2.set(MRJobConfig.TASK_ATTEMPT_ID, attempt0);
+ conf2.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 2);
+ JobContext jContext2 = new JobContextImpl(conf2, taskAttempt0.getJobID());
+ TaskAttemptContext tContext2 = new TaskAttemptContextImpl(conf2,
+ taskAttempt0);
+ AbstractS3ACommitter committer2 = createCommitter(tContext2);
+ committer2.setupJob(tContext2);
+
+ assertFalse("recoverySupported in " + committer2,
+ committer2.isRecoverySupported());
+ intercept(PathCommitException.class, "recover",
+ () -> committer2.recoverTask(tContext2));
+
+ // at this point, task attempt 0 has failed to recover
+ // it should be abortable though. This will be a no-op as it already
+ // committed
+ describe("aborting task attempt 2; expect nothing to clean up");
+ committer2.abortTask(tContext2);
+ describe("Aborting job 2; expect pending commits to be aborted");
+ committer2.abortJob(jContext2, JobStatus.State.KILLED);
+ // now, state of system may still have pending data
+ assertNoMultipartUploadsPending(outDir);
+ }
+
+ protected void assertTaskAttemptPathDoesNotExist(
+ AbstractS3ACommitter committer, TaskAttemptContext context)
+ throws IOException {
+ Path attemptPath = committer.getTaskAttemptPath(context);
+ ContractTestUtils.assertPathDoesNotExist(
+ attemptPath.getFileSystem(context.getConfiguration()),
+ "task attempt dir",
+ attemptPath);
+ }
+
+ protected void assertJobAttemptPathDoesNotExist(
+ AbstractS3ACommitter committer, JobContext context)
+ throws IOException {
+ Path attemptPath = committer.getJobAttemptPath(context);
+ ContractTestUtils.assertPathDoesNotExist(
+ attemptPath.getFileSystem(context.getConfiguration()),
+ "job attempt dir",
+ attemptPath);
+ }
+
+ /**
+ * Verify the output of the directory.
+ * That includes the {@code part-m-00000-*}
+ * file existence and contents, as well as optionally, the success marker.
+ * @param dir directory to scan.
+ * @param expectSuccessMarker check the success marker?
+ * @throws Exception failure.
+ */
+ private void validateContent(Path dir, boolean expectSuccessMarker)
+ throws Exception {
+ if (expectSuccessMarker) {
+ verifySuccessMarker(dir);
+ }
+ Path expectedFile = getPart0000(dir);
+ log().debug("Validating content in {}", expectedFile);
+ StringBuffer expectedOutput = new StringBuffer();
+ expectedOutput.append(KEY_1).append('\t').append(VAL_1).append("\n");
+ expectedOutput.append(VAL_1).append("\n");
+ expectedOutput.append(VAL_2).append("\n");
+ expectedOutput.append(KEY_2).append("\n");
+ expectedOutput.append(KEY_1).append("\n");
+ expectedOutput.append(KEY_2).append('\t').append(VAL_2).append("\n");
+ String output = readFile(expectedFile);
+ assertEquals("Content of " + expectedFile,
+ expectedOutput.toString(), output);
+ }
+
+ /**
+ * Identify any path under the directory which begins with the
+ * {@code "part-m-00000"} sequence. There's some compensation for
+ * eventual consistency here.
+ * @param dir directory to scan
+ * @return the full path
+ * @throws FileNotFoundException the path is missing.
+ * @throws Exception failure.
+ */
+ protected Path getPart0000(final Path dir) throws Exception {
+ final FileSystem fs = dir.getFileSystem(getConfiguration());
+ return eventually(CONSISTENCY_WAIT, CONSISTENCY_PROBE_INTERVAL,
+ () -> getPart0000Immediately(fs, dir));
+ }
+
+ /**
+ * Identify any path under the directory which begins with the
+ * {@code "part-m-00000"} sequence. There's some compensation for
+ * eventual consistency here.
+ * @param fs FS to probe
+ * @param dir directory to scan
+ * @return the full path
+ * @throws FileNotFoundException the path is missing.
+ * @throws IOException failure.
+ */
+ private Path getPart0000Immediately(FileSystem fs, Path dir)
+ throws IOException {
+ FileStatus[] statuses = fs.listStatus(dir,
+ path -> path.getName().startsWith(PART_00000));
+ if (statuses.length != 1) {
+ // fail, with a listing of the parent dir
+ ContractTestUtils.assertPathExists(fs, "Output file",
+ new Path(dir, PART_00000));
+ }
+ return statuses[0].getPath();
+ }
+
+ /**
+ * Look for the partFile subdir of the output dir.
+ * @param fs filesystem
+ * @param dir output dir
+ * @throws Exception failure.
+ */
+ private void validateMapFileOutputContent(
+ FileSystem fs, Path dir) throws Exception {
+ // map output is a directory with index and data files
+ assertPathExists("Map output", dir);
+ Path expectedMapDir = getPart0000(dir);
+ assertPathExists("Map output", expectedMapDir);
+ assertIsDirectory(expectedMapDir);
+ FileStatus[] files = fs.listStatus(expectedMapDir);
+ assertTrue("No files found in " + expectedMapDir, files.length > 0);
+ assertPathExists("index file in " + expectedMapDir,
+ new Path(expectedMapDir, MapFile.INDEX_FILE_NAME));
+ assertPathExists("data file in " + expectedMapDir,
+ new Path(expectedMapDir, MapFile.DATA_FILE_NAME));
+ }
+
+ /**
+ * Dump all MPUs in the filesystem.
+ * @throws IOException IO failure
+ */
+ protected void dumpMultipartUploads() throws IOException {
+ countMultipartUploads("");
+ }
+
+ /**
+ * Full test of the expected lifecycle: start job, task, write, commit task,
+ * commit job.
+ * @throws Exception on a failure
+ */
+ @Test
+ public void testCommitLifecycle() throws Exception {
+ describe("Full test of the expected lifecycle:\n" +
+ " start job, task, write, commit task, commit job.\n" +
+ "Verify:\n" +
+ "* no files are visible after task commit\n" +
+ "* the expected file is visible after job commit\n" +
+ "* no outstanding MPUs after job commit");
+ JobData jobData = startJob(false);
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+
+ // write output
+ describe("1. Writing output");
+ writeTextOutput(tContext);
+
+ dumpMultipartUploads();
+ describe("2. Committing task");
+ assertTrue("No files to commit were found by " + committer,
+ committer.needsTaskCommit(tContext));
+ committer.commitTask(tContext);
+
+ // this is only task commit; there MUST be no part- files in the dest dir
+ waitForConsistency();
+ try {
+ applyLocatedFiles(getFileSystem().listFiles(outDir, false),
+ (status) ->
+ assertFalse("task committed file to dest :" + status,
+ status.getPath().toString().contains("part")));
+ } catch (FileNotFoundException ignored) {
+ log().info("Outdir {} is not created by task commit phase ",
+ outDir);
+ }
+
+ describe("3. Committing job");
+ assertMultipartUploadsPending(outDir);
+ committer.commitJob(jContext);
+
+ // validate output
+ describe("4. Validating content");
+ validateContent(outDir, shouldExpectSuccessMarker());
+ assertNoMultipartUploadsPending(outDir);
+ }
+
+ @Test
+ public void testCommitterWithDuplicatedCommit() throws Exception {
+ describe("Call a task then job commit twice;" +
+ "expect the second task commit to fail.");
+ JobData jobData = startJob(true);
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+
+ // do commit
+ commit(committer, jContext, tContext);
+
+ // validate output
+ validateContent(outDir, shouldExpectSuccessMarker());
+
+ assertNoMultipartUploadsPending(outDir);
+
+ // commit task to fail on retry
+ expectFNFEonTaskCommit(committer, tContext);
+ }
+
+ protected boolean shouldExpectSuccessMarker() {
+ return true;
+ }
+
+ /**
+ * Simulate a failure on the first job commit; expect the
+ * second to succeed.
+ */
+ @Test
+ public void testCommitterWithFailure() throws Exception {
+ describe("Fail the first job commit then retry");
+ JobData jobData = startJob(new FailingCommitterFactory(), true);
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+
+ // do commit
+ committer.commitTask(tContext);
+
+ // now fail job
+ expectSimulatedFailureOnJobCommit(jContext, committer);
+
+ committer.commitJob(jContext);
+
+ // but the data got there, due to the order of operations.
+ validateContent(outDir, shouldExpectSuccessMarker());
+ expectJobCommitToFail(jContext, committer);
+ }
+
+ /**
+ * Override point: the failure expected on the attempt to commit a failed
+ * job.
+ * @param jContext job context
+ * @param committer committer
+ * @throws Exception any unexpected failure.
+ */
+ protected void expectJobCommitToFail(JobContext jContext,
+ AbstractS3ACommitter committer) throws Exception {
+ // next attempt will fail as there is no longer a directory to commit
+ expectJobCommitFailure(jContext, committer,
+ FileNotFoundException.class);
+ }
+
+ /**
+ * Expect a job commit operation to fail with a specific exception.
+ * @param jContext job context
+ * @param committer committer
+ * @param clazz class of exception
+ * @return the caught exception
+ * @throws Exception any unexpected failure.
+ */
+ protected static <E extends IOException> E expectJobCommitFailure(
+ JobContext jContext,
+ AbstractS3ACommitter committer,
+ Class<E> clazz)
+ throws Exception {
+
+ return intercept(clazz,
+ () -> {
+ committer.commitJob(jContext);
+ return committer.toString();
+ });
+ }
+
+ protected static void expectFNFEonTaskCommit(
+ AbstractS3ACommitter committer,
+ TaskAttemptContext tContext) throws Exception {
+ intercept(FileNotFoundException.class,
+ () -> {
+ committer.commitTask(tContext);
+ return committer.toString();
+ });
+ }
+
+ /**
+ * Simulate a failure on the first job commit; expect the
+ * second to succeed.
+ */
+ @Test
+ public void testCommitterWithNoOutputs() throws Exception {
+ describe("Have a task and job with no outputs: expect success");
+ JobData jobData = startJob(new FailingCommitterFactory(), false);
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+
+ // do commit
+ committer.commitTask(tContext);
+ assertTaskAttemptPathDoesNotExist(committer, tContext);
+ }
+
+ protected static void expectSimulatedFailureOnJobCommit(JobContext jContext,
+ AbstractS3ACommitter committer) throws Exception {
+ ((CommitterFaultInjection) committer).setFaults(
+ CommitterFaultInjection.Faults.commitJob);
+ expectJobCommitFailure(jContext, committer,
+ CommitterFaultInjectionImpl.Failure.class);
+ }
+
+ @Test
+ public void testMapFileOutputCommitter() throws Exception {
+ describe("Test that the committer generates map output into a directory\n" +
+ "starting with the prefix part-");
+ JobData jobData = startJob(false);
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+ Configuration conf = jobData.conf;
+
+ // write output
+ writeMapFileOutput(new MapFileOutputFormat().getRecordWriter(tContext),
+ tContext);
+
+ // do commit
+ commit(committer, jContext, tContext);
+ S3AFileSystem fs = getFileSystem();
+ waitForConsistency();
+ lsR(fs, outDir, true);
+ String ls = ls(outDir);
+ describe("\nvalidating");
+
+ // validate output
+ verifySuccessMarker(outDir);
+
+ describe("validate output of %s", outDir);
+ validateMapFileOutputContent(fs, outDir);
+
+ // Ensure getReaders call works and also ignores
+ // hidden filenames (_ or . prefixes)
+ describe("listing");
+ FileStatus[] filtered = fs.listStatus(outDir, HIDDEN_FILE_FILTER);
+ assertEquals("listed children under " + ls,
+ 1, filtered.length);
+ FileStatus fileStatus = filtered[0];
+ assertTrue("Not the part file: " + fileStatus,
+ fileStatus.getPath().getName().startsWith(PART_00000));
+
+ describe("getReaders()");
+ assertEquals("Number of MapFile.Reader entries with shared FS "
+ + outDir + " : " + ls,
+ 1, getReaders(fs, outDir, conf).length);
+
+ describe("getReaders(new FS)");
+ FileSystem fs2 = FileSystem.get(outDir.toUri(), conf);
+ assertEquals("Number of MapFile.Reader entries with shared FS2 "
+ + outDir + " : " + ls,
+ 1, getReaders(fs2, outDir, conf).length);
+
+ describe("MapFileOutputFormat.getReaders");
+ assertEquals("Number of MapFile.Reader entries with new FS in "
+ + outDir + " : " + ls,
+ 1, MapFileOutputFormat.getReaders(outDir, conf).length);
+ }
+
+ /** Open the output generated by this format. */
+ @SuppressWarnings("IOResourceOpenedButNotSafelyClosed")
+ private static MapFile.Reader[] getReaders(FileSystem fs,
+ Path dir,
+ Configuration conf) throws IOException {
+ Path[] names = FileUtil.stat2Paths(fs.listStatus(dir, HIDDEN_FILE_FILTER));
+
+ // sort names, so that hash partitioning works
+ Arrays.sort(names);
+
+ MapFile.Reader[] parts = new MapFile.Reader[names.length];
+ for (int i = 0; i < names.length; i++) {
+ parts[i] = new MapFile.Reader(names[i], conf);
+ }
+ return parts;
+ }
+
+ /**
+ * A functional interface which an action to test must implement.
+ */
+ @FunctionalInterface
+ public interface ActionToTest {
+ void exec(Job job, JobContext jContext, TaskAttemptContext tContext,
+ AbstractS3ACommitter committer) throws Exception;
+ }
+
+ @Test
+ public void testAbortTaskNoWorkDone() throws Exception {
+ executeWork("abort task no work",
+ (job, jContext, tContext, committer) ->
+ committer.abortTask(tContext));
+ }
+
+ @Test
+ public void testAbortJobNoWorkDone() throws Exception {
+ executeWork("abort task no work",
+ (job, jContext, tContext, committer) ->
+ committer.abortJob(jContext, JobStatus.State.RUNNING));
+ }
+
+ @Test
+ public void testCommitJobButNotTask() throws Exception {
+ executeWork("commit a job while a task's work is pending, " +
+ "expect task writes to be cancelled.",
+ (job, jContext, tContext, committer) -> {
+ // step 1: write the text
+ writeTextOutput(tContext);
+ // step 2: commit the job
+ createCommitter(tContext).commitJob(tContext);
+ // verify that no output can be observed
+ assertPart0000DoesNotExist(outDir);
+ // that includes, no pending MPUs; commitJob is expected to
+ // cancel any.
+ assertNoMultipartUploadsPending(outDir);
+ }
+ );
+ }
+
+ @Test
+ public void testAbortTaskThenJob() throws Exception {
+ JobData jobData = startJob(true);
+ AbstractS3ACommitter committer = jobData.committer;
+
+ // do abort
+ committer.abortTask(jobData.tContext);
+
+ intercept(FileNotFoundException.class, "",
+ () -> getPart0000(committer.getWorkPath()));
+
+ committer.abortJob(jobData.jContext, JobStatus.State.FAILED);
+ assertJobAbortCleanedUp(jobData);
+ }
+
+ /**
+ * Extension point: assert that the job was all cleaned up after an abort.
+ * Base assertions
+ * <ul>
+ * <li>Output dir is absent or, if present, empty</li>
+ * <li>No pending MPUs to/under the output dir</li>
+ * </ul>
+ * @param jobData job data
+ * @throws Exception failure
+ */
+ public void assertJobAbortCleanedUp(JobData jobData) throws Exception {
+ // special handling of magic directory; harmless in staging
+ S3AFileSystem fs = getFileSystem();
+ try {
+ FileStatus[] children = listChildren(fs, outDir);
+ if (children.length != 0) {
+ lsR(fs, outDir, true);
+ }
+ assertArrayEquals("Output directory not empty " + ls(outDir),
+ new FileStatus[0], children);
+ } catch (FileNotFoundException e) {
+ // this is a valid failure mode; it means the dest dir doesn't exist yet.
+ }
+ assertNoMultipartUploadsPending(outDir);
+ }
+
+ @Test
+ public void testFailAbort() throws Exception {
+ describe("Abort the task, then job (failed), abort the job again");
+ JobData jobData = startJob(true);
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+ AbstractS3ACommitter committer = jobData.committer;
+
+ // do abort
+ committer.abortTask(tContext);
+
+ committer.getJobAttemptPath(jContext);
+ committer.getTaskAttemptPath(tContext);
+ assertPart0000DoesNotExist(outDir);
+ assertSuccessMarkerDoesNotExist(outDir);
+ describe("Aborting job into %s", outDir);
+
+ committer.abortJob(jContext, JobStatus.State.FAILED);
+
+ assertTaskAttemptPathDoesNotExist(committer, tContext);
+ assertJobAttemptPathDoesNotExist(committer, jContext);
+
+ // try again; expect abort to be idempotent.
+ committer.abortJob(jContext, JobStatus.State.FAILED);
+ assertNoMultipartUploadsPending(outDir);
+ }
+
+ public void assertPart0000DoesNotExist(Path dir) throws Exception {
+ intercept(FileNotFoundException.class,
+ () -> getPart0000(dir));
+ assertPathDoesNotExist("expected output file", new Path(dir, PART_00000));
+ }
+
+ @Test
+ public void testAbortJobNotTask() throws Exception {
+ executeWork("abort task no work",
+ (job, jContext, tContext, committer) -> {
+ // write output
+ writeTextOutput(tContext);
+ committer.abortJob(jContext, JobStatus.State.RUNNING);
+ assertTaskAttemptPathDoesNotExist(
+ committer, tContext);
+ assertJobAttemptPathDoesNotExist(
+ committer, jContext);
+ assertNoMultipartUploadsPending(outDir);
+ });
+ }
+
+ /**
+ * This looks at what happens with concurrent commits.
+ * However, the failure condition it looks for (subdir under subdir)
+ * is the kind of failure you see on a rename-based commit.
+ *
+ * What it will not detect is the fact that both tasks will each commit
+ * to the destination directory. That is: whichever commits last wins.
+ *
+ * There's no way to stop this. Instead it is a requirement that the task
+ * commit operation is only executed when the committer is happy to
+ * commit only those tasks which it knows have succeeded, and abort those
+ * which have not.
+ * @throws Exception failure
+ */
+ @Test
+ public void testConcurrentCommitTaskWithSubDir() throws Exception {
+ Job job = newJob();
+ FileOutputFormat.setOutputPath(job, outDir);
+ final Configuration conf = job.getConfiguration();
+
+ final JobContext jContext =
+ new JobContextImpl(conf, taskAttempt0.getJobID());
+ AbstractS3ACommitter amCommitter = createCommitter(
+ new TaskAttemptContextImpl(conf, taskAttempt0));
+ amCommitter.setupJob(jContext);
+
+ final TaskAttemptContext[] taCtx = new TaskAttemptContextImpl[2];
+ taCtx[0] = new TaskAttemptContextImpl(conf, taskAttempt0);
+ taCtx[1] = new TaskAttemptContextImpl(conf, taskAttempt1);
+
+ final TextOutputFormat[] tof = new LoggingTextOutputFormat[2];
+ for (int i = 0; i < tof.length; i++) {
+ tof[i] = new LoggingTextOutputFormat() {
+ @Override
+ public Path getDefaultWorkFile(
+ TaskAttemptContext context,
+ String extension) throws IOException {
+ final AbstractS3ACommitter foc = (AbstractS3ACommitter)
+ getOutputCommitter(context);
+ return new Path(new Path(foc.getWorkPath(), SUB_DIR),
+ getUniqueFile(context, getOutputName(context), extension));
+ }
+ };
+ }
+
+ final ExecutorService executor = HadoopExecutors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < taCtx.length; i++) {
+ final int taskIdx = i;
+ executor.submit(() -> {
+ final OutputCommitter outputCommitter =
+ tof[taskIdx].getOutputCommitter(taCtx[taskIdx]);
+ outputCommitter.setupTask(taCtx[taskIdx]);
+ final RecordWriter rw =
+ tof[taskIdx].getRecordWriter(taCtx[taskIdx]);
+ writeOutput(rw, taCtx[taskIdx]);
+ describe("Committing Task %d", taskIdx);
+ outputCommitter.commitTask(taCtx[taskIdx]);
+ return null;
+ });
+ }
+ } finally {
+ executor.shutdown();
+ while (!executor.awaitTermination(1, TimeUnit.SECONDS)) {
+ log().info("Awaiting thread termination!");
+ }
+ }
+
+ // if we commit here then all tasks will be committed, so there will
+ // be contention for that final directory: both parts will go in.
+
+ describe("\nCommitting Job");
+ amCommitter.commitJob(jContext);
+ assertPathExists("base output directory", outDir);
+ assertPart0000DoesNotExist(outDir);
+ Path outSubDir = new Path(outDir, SUB_DIR);
+ assertPathDoesNotExist("Must not end up with sub_dir/sub_dir",
+ new Path(outSubDir, SUB_DIR));
+
+ // validate output
+ // There's no success marker in the subdirectory
+ validateContent(outSubDir, false);
+ }
+
+ /**
+ * Create a committer which fails; the class
+ * {@link CommitterFaultInjectionImpl} implements the logic.
+ * @param tContext task context
+ * @return committer instance
+ * @throws IOException failure to instantiate
+ */
+ protected abstract AbstractS3ACommitter createFailingCommitter(
+ TaskAttemptContext tContext) throws IOException;
+
+ /**
+ * Factory for failing committers.
+ */
+ public class FailingCommitterFactory implements CommitterFactory {
+ @Override
+ public AbstractS3ACommitter createCommitter(TaskAttemptContext context)
+ throws IOException {
+ return createFailingCommitter(context);
+ }
+ }
+
+ @Test
+ public void testOutputFormatIntegration() throws Throwable {
+ Configuration conf = getConfiguration();
+ Job job = newJob();
+ job.setOutputFormatClass(LoggingTextOutputFormat.class);
+ conf = job.getConfiguration();
+ conf.set(MRJobConfig.TASK_ATTEMPT_ID, attempt0);
+ conf.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
+ JobContext jContext = new JobContextImpl(conf, taskAttempt0.getJobID());
+ TaskAttemptContext tContext = new TaskAttemptContextImpl(conf,
+ taskAttempt0);
+ LoggingTextOutputFormat outputFormat = (LoggingTextOutputFormat)
+ ReflectionUtils.newInstance(tContext.getOutputFormatClass(), conf);
+ AbstractS3ACommitter committer = (AbstractS3ACommitter)
+ outputFormat.getOutputCommitter(tContext);
+
+ // setup
+ JobData jobData = new JobData(job, jContext, tContext, committer);
+ setup(jobData);
+ abortInTeardown(jobData);
+ LoggingTextOutputFormat.LoggingLineRecordWriter recordWriter
+ = outputFormat.getRecordWriter(tContext);
+ IntWritable iw = new IntWritable(1);
+ recordWriter.write(iw, iw);
+ Path dest = recordWriter.getDest();
+ validateTaskAttemptPathDuringWrite(dest);
+ recordWriter.close(tContext);
+ // at this point
+ validateTaskAttemptPathAfterWrite(dest);
+ assertTrue("Committer does not have data to commit " + committer,
+ committer.needsTaskCommit(tContext));
+ committer.commitTask(tContext);
+ committer.commitJob(jContext);
+ // validate output
+ verifySuccessMarker(outDir);
+ }
+
+ /**
+ * Create a committer through reflection then use it to abort
+ * a task. This mimics the action of an AM when a container fails and
+ * the AM wants to abort the task attempt.
+ */
+ @Test
+ public void testAMWorkflow() throws Throwable {
+ describe("Create a committer with a null output path & use as an AM");
+ JobData jobData = startJob(true);
+ JobContext jContext = jobData.jContext;
+ TaskAttemptContext tContext = jobData.tContext;
+
+ TaskAttemptContext newAttempt = taskAttemptForJob(
+ MRBuilderUtils.newJobId(1, 1, 1), jContext);
+ Configuration conf = jContext.getConfiguration();
+
+ // bind
+ LoggingTextOutputFormat.bind(conf);
+
+ OutputFormat<?, ?> outputFormat
+ = ReflectionUtils.newInstance(newAttempt
+ .getOutputFormatClass(), conf);
+ Path outputPath = FileOutputFormat.getOutputPath(newAttempt);
+ assertNotNull("null output path in new task attempt", outputPath);
+
+ AbstractS3ACommitter committer2 = (AbstractS3ACommitter)
+ outputFormat.getOutputCommitter(newAttempt);
+ committer2.abortTask(tContext);
+ assertNoMultipartUploadsPending(getOutDir());
+ }
+
+
+ @Test
+ public void testParallelJobsToAdjacentPaths() throws Throwable {
+
+ describe("Run two jobs in parallel, assert they both complete");
+ JobData jobData = startJob(true);
+ Job job1 = jobData.job;
+ AbstractS3ACommitter committer1 = jobData.committer;
+ JobContext jContext1 = jobData.jContext;
+ TaskAttemptContext tContext1 = jobData.tContext;
+
+ // now build up a second job
+ String jobId2 = randomJobId();
+ String attempt20 = "attempt_" + jobId2 + "_m_000000_0";
+ TaskAttemptID taskAttempt20 = TaskAttemptID.forName(attempt20);
+ String attempt21 = "attempt_" + jobId2 + "_m_000001_0";
+ TaskAttemptID taskAttempt21 = TaskAttemptID.forName(attempt21);
+
+ Path job1Dest = outDir;
+ Path job2Dest = new Path(getOutDir().getParent(),
+ getMethodName() + "job2Dest");
+ // little safety check
+ assertNotEquals(job1Dest, job2Dest);
+
+ // create the second job
+ Job job2 = newJob(job2Dest, new JobConf(getConfiguration()), attempt20);
+ Configuration conf2 = job2.getConfiguration();
+ conf2.setInt(MRJobConfig.APPLICATION_ATTEMPT_ID, 1);
+ try {
+ JobContext jContext2 = new JobContextImpl(conf2,
+ taskAttempt20.getJobID());
+ TaskAttemptContext tContext2 =
+ new TaskAttemptContextImpl(conf2, taskAttempt20);
+ AbstractS3ACommitter committer2 = createCommitter(job2Dest, tContext2);
+ JobData jobData2 = new JobData(job2, jContext2, tContext2, committer2);
+ setup(jobData2);
+ abortInTeardown(jobData2);
+ // make sure the directories are different
+ assertEquals(job2Dest, committer2.getOutputPath());
+
+ // job2 setup, write some data there
+ writeTextOutput(tContext2);
+
+ // at this point, job1 and job2 both have uncommitted tasks
+
+ // commit tasks in order task 2, task 1.
+ committer2.commitTask(tContext2);
+ committer1.commitTask(tContext1);
+
+ assertMultipartUploadsPending(job1Dest);
+ assertMultipartUploadsPending(job2Dest);
+
+ // commit jobs in order job 1, job 2
+ committer1.commitJob(jContext1);
+ assertNoMultipartUploadsPending(job1Dest);
+ getPart0000(job1Dest);
+ assertMultipartUploadsPending(job2Dest);
+
+ committer2.commitJob(jContext2);
+ getPart0000(job2Dest);
+ assertNoMultipartUploadsPending(job2Dest);
+ } finally {
+ // uncommitted files to this path need to be deleted in tests which fail
+ abortMultipartUploadsUnderPath(job2Dest);
+ }
+
+ }
+
+ protected void validateTaskAttemptPathDuringWrite(Path p) throws IOException {
+
+ }
+
+ protected void validateTaskAttemptPathAfterWrite(Path p) throws IOException {
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjection.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjection.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjection.java
new file mode 100644
index 0000000..7dd1b0a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjection.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+/**
+ * Support for adding fault injection: all the failing committers in the IT
+ * tests must implement this.
+ */
+public interface CommitterFaultInjection {
+ String COMMIT_FAILURE_MESSAGE = "oops";
+
+ void setFaults(Faults... faults);
+
+ /**
+ * Operations which can fail.
+ */
+ enum Faults {
+ abortJob,
+ abortTask,
+ cleanupJob,
+ commitJob,
+ commitTask,
+ getWorkPath,
+ needsTaskCommit,
+ setupJob,
+ setupTask,
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjectionImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjectionImpl.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjectionImpl.java
new file mode 100644
index 0000000..d423398
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/CommitterFaultInjectionImpl.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+
+/**
+ * Implementation of the fault injection lifecycle.
+ * Can reset a fault on failure or always raise it.
+ */
+public final class CommitterFaultInjectionImpl
+ extends PathOutputCommitter implements CommitterFaultInjection {
+
+ private Set<Faults> faults;
+ private boolean resetOnFailure;
+
+ public CommitterFaultInjectionImpl(Path outputPath,
+ JobContext context,
+ boolean resetOnFailure,
+ Faults... faults) throws IOException {
+ super(outputPath, context);
+ setFaults(faults);
+ this.resetOnFailure = resetOnFailure;
+ }
+
+ @Override
+ public void setFaults(Faults... faults) {
+ this.faults = new HashSet<>(faults.length);
+ Collections.addAll(this.faults, faults);
+ }
+
+ /**
+ * Fail if the condition is in the set of faults, may optionally reset
+ * it before failing.
+ * @param condition condition to check for
+ * @throws Failure if the condition is faulting
+ */
+ private void maybeFail(Faults condition) throws Failure {
+ if (faults.contains(condition)) {
+ if (resetOnFailure) {
+ faults.remove(condition);
+ }
+ throw new Failure();
+ }
+ }
+
+ @Override
+ public Path getWorkPath() throws IOException {
+ maybeFail(Faults.getWorkPath);
+ return null;
+ }
+
+ @Override
+ public Path getOutputPath() {
+ return null;
+ }
+
+
+ @Override
+ public void setupJob(JobContext jobContext) throws IOException {
+ maybeFail(Faults.setupJob);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskContext) throws IOException {
+ maybeFail(Faults.setupTask);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskContext)
+ throws IOException {
+ maybeFail(Faults.needsTaskCommit);
+ return false;
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskContext) throws IOException {
+ maybeFail(Faults.commitTask);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskContext) throws IOException {
+ maybeFail(Faults.abortTask);
+ }
+
+ @Override
+ public void commitJob(JobContext jobContext) throws IOException {
+ maybeFail(Faults.commitJob);
+ }
+
+ @Override
+ public void abortJob(JobContext jobContext, JobStatus.State state)
+ throws IOException {
+ maybeFail(Faults.abortJob);
+ }
+
+ /**
+ * The exception raised on failure.
+ */
+ public static class Failure extends IOException {
+ public Failure() {
+ super(COMMIT_FAILURE_MESSAGE);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
new file mode 100644
index 0000000..2a98382
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/ITestCommitOperations.java
@@ -0,0 +1,545 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+
+import com.amazonaws.services.s3.model.PartETag;
+import org.junit.Assume;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker;
+import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitter;
+import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitterFactory;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.*;
+
+/**
+ * Test the low-level binding of the S3A FS to the magic commit mechanism,
+ * and handling of the commit operations.
+ * This is done with an inconsistent client.
+ */
+public class ITestCommitOperations extends AbstractCommitITest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ITestCommitOperations.class);
+ private static final byte[] DATASET = dataset(1000, 'a', 32);
+ private static final String S3A_FACTORY_KEY = String.format(
+ COMMITTER_FACTORY_SCHEME_PATTERN, "s3a");
+
+ /**
+ * A compile time flag which allows you to disable failure reset before
+ * assertions and teardown.
+ * As S3A is now required to be resilient to failure on all FS operations,
+ * setting it to false ensures that even the assertions are checking
+ * the resilience codepaths.
+ */
+ private static final boolean RESET_FAILURES_ENABLED = false;
+
+ private static final float HIGH_THROTTLE = 0.25f;
+
+ private static final float FULL_THROTTLE = 1.0f;
+
+ private static final int STANDARD_FAILURE_LIMIT = 2;
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ bindCommitter(conf, CommitConstants.S3A_COMMITTER_FACTORY,
+ CommitConstants.COMMITTER_NAME_MAGIC);
+ return conf;
+ }
+
+ @Override
+ public boolean useInconsistentClient() {
+ return true;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ FileSystem.closeAll();
+ super.setup();
+ verifyIsMagicCommitFS(getFileSystem());
+ // abort,; rethrow on failure
+ setThrottling(HIGH_THROTTLE, STANDARD_FAILURE_LIMIT);
+ }
+
+ @Test
+ public void testCreateTrackerNormalPath() throws Throwable {
+ S3AFileSystem fs = getFileSystem();
+ MagicCommitIntegration integration
+ = new MagicCommitIntegration(fs, true);
+ String filename = "notdelayed.txt";
+ Path destFile = methodPath(filename);
+ String origKey = fs.pathToKey(destFile);
+ PutTracker tracker = integration.createTracker(destFile, origKey);
+ assertFalse("wrong type: " + tracker + " for " + destFile,
+ tracker instanceof MagicCommitTracker);
+ }
+
+ /**
+ * On a magic path, the magic tracker is returned.
+ * @throws Throwable failure
+ */
+ @Test
+ public void testCreateTrackerMagicPath() throws Throwable {
+ S3AFileSystem fs = getFileSystem();
+ MagicCommitIntegration integration
+ = new MagicCommitIntegration(fs, true);
+ String filename = "delayed.txt";
+ Path destFile = methodPath(filename);
+ String origKey = fs.pathToKey(destFile);
+ Path pendingPath = makeMagic(destFile);
+ verifyIsMagicCommitPath(fs, pendingPath);
+ String pendingPathKey = fs.pathToKey(pendingPath);
+ assertTrue("wrong path of " + pendingPathKey,
+ pendingPathKey.endsWith(filename));
+ final List<String> elements = splitPathToElements(pendingPath);
+ assertEquals("splitPathToElements()", filename, lastElement(elements));
+ List<String> finalDestination = finalDestination(elements);
+ assertEquals("finalDestination()",
+ filename,
+ lastElement(finalDestination));
+ final String destKey = elementsToKey(finalDestination);
+ assertEquals("destination key", origKey, destKey);
+
+ PutTracker tracker = integration.createTracker(pendingPath,
+ pendingPathKey);
+ assertTrue("wrong type: " + tracker + " for " + pendingPathKey,
+ tracker instanceof MagicCommitTracker);
+ assertEquals("tracker destination key", origKey, tracker.getDestKey());
+
+ Path pendingSuffixedPath = new Path(pendingPath,
+ "part-0000" + PENDING_SUFFIX);
+ assertFalse("still a delayed complete path " + pendingSuffixedPath,
+ fs.isMagicCommitPath(pendingSuffixedPath));
+ Path pendingSet = new Path(pendingPath,
+ "part-0000" + PENDINGSET_SUFFIX);
+ assertFalse("still a delayed complete path " + pendingSet,
+ fs.isMagicCommitPath(pendingSet));
+ }
+
+ @Test
+ public void testCreateAbortEmptyFile() throws Throwable {
+ describe("create then abort an empty file; throttled");
+ S3AFileSystem fs = getFileSystem();
+ String filename = "empty-abort.txt";
+ Path destFile = methodPath(filename);
+ Path pendingFilePath = makeMagic(destFile);
+ touch(fs, pendingFilePath);
+ validateIntermediateAndFinalPaths(pendingFilePath, destFile);
+ Path pendingDataPath = validatePendingCommitData(filename,
+ pendingFilePath);
+
+ CommitOperations actions = newCommitOperations();
+ // abort,; rethrow on failure
+ fullThrottle();
+ LOG.info("Abort call");
+ actions.abortAllSinglePendingCommits(pendingDataPath.getParent(), true)
+ .maybeRethrow();
+ resetFailures();
+ assertPathDoesNotExist("pending file not deleted", pendingDataPath);
+ assertPathDoesNotExist("dest file was created", destFile);
+ }
+
+ private void fullThrottle() {
+ setThrottling(FULL_THROTTLE, STANDARD_FAILURE_LIMIT);
+ }
+
+ private CommitOperations newCommitOperations() {
+ return new CommitOperations(getFileSystem());
+ }
+
+ @Override
+ protected void resetFailures() {
+ if (!RESET_FAILURES_ENABLED) {
+ super.resetFailures();
+ }
+ }
+
+ /**
+ * Create a new path which has the same filename as the dest file, but
+ * is in a magic directory under the destination dir.
+ * @param destFile final destination file
+ * @return magic path
+ */
+ private static Path makeMagic(Path destFile) {
+ return new Path(destFile.getParent(),
+ MAGIC + '/' + destFile.getName());
+ }
+
+ @Test
+ public void testCommitEmptyFile() throws Throwable {
+ describe("create then commit an empty file");
+ createCommitAndVerify("empty-commit.txt", new byte[0]);
+ }
+
+ @Test
+ public void testCommitSmallFile() throws Throwable {
+ describe("create then commit an empty file");
+ createCommitAndVerify("small-commit.txt", DATASET);
+ }
+
+ @Test
+ public void testAbortNonexistentDir() throws Throwable {
+ describe("Attempt to abort a directory that does not exist");
+ Path destFile = methodPath("testAbortNonexistentPath");
+ newCommitOperations()
+ .abortAllSinglePendingCommits(destFile, true)
+ .maybeRethrow();
+ }
+
+ @Test
+ public void testCommitterFactoryDefault() throws Throwable {
+ Configuration conf = new Configuration();
+ Path dest = methodPath();
+ conf.set(COMMITTER_FACTORY_CLASS,
+ MagicS3GuardCommitterFactory.CLASSNAME);
+ PathOutputCommitterFactory factory
+ = getCommitterFactory(dest, conf);
+ PathOutputCommitter committer = factory.createOutputCommitter(
+ methodPath(),
+ new TaskAttemptContextImpl(getConfiguration(),
+ new TaskAttemptID(new TaskID(), 1)));
+ assertEquals("Wrong committer",
+ MagicS3GuardCommitter.class, committer.getClass());
+ }
+
+ @Test
+ public void testCommitterFactorySchema() throws Throwable {
+ Configuration conf = new Configuration();
+ Path dest = methodPath();
+
+ conf.set(S3A_FACTORY_KEY,
+ MagicS3GuardCommitterFactory.CLASSNAME);
+ PathOutputCommitterFactory factory
+ = getCommitterFactory(dest, conf);
+ // the casting is an implicit type check
+ MagicS3GuardCommitter s3a = (MagicS3GuardCommitter)
+ factory.createOutputCommitter(
+ methodPath(),
+ new TaskAttemptContextImpl(getConfiguration(),
+ new TaskAttemptID(new TaskID(), 1)));
+ // should never get here
+ assertNotNull(s3a);
+ }
+
+ @Test
+ public void testBaseRelativePath() throws Throwable {
+ describe("Test creating file with a __base marker and verify that it ends" +
+ " up in where expected");
+ Path destDir = methodPath("testBaseRelativePath");
+ Path pendingBaseDir = new Path(destDir, MAGIC + "/child/" + BASE);
+ String child = "subdir/child.txt";
+ Path pendingChildPath = new Path(pendingBaseDir, child);
+ Path expectedDestPath = new Path(destDir, child);
+ createFile(getFileSystem(), pendingChildPath, true, DATASET);
+ commit("child.txt", pendingChildPath, expectedDestPath, 0, 0);
+ }
+
+ private void createCommitAndVerify(String filename, byte[] data)
+ throws Exception {
+ S3AFileSystem fs = getFileSystem();
+ Path destFile = methodPath(filename);
+ Path magicDest = makeMagic(destFile);
+ try(FSDataOutputStream stream = fs.create(magicDest, true)) {
+ assertTrue(stream.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT));
+ if (data != null && data.length > 0) {
+ stream.write(data);
+ }
+ stream.close();
+ }
+ FileStatus status = getFileStatusEventually(fs, magicDest,
+ CONSISTENCY_WAIT);
+ assertEquals("Non empty marker file: " + status, 0, status.getLen());
+
+ commit(filename, destFile, HIGH_THROTTLE, 0);
+ verifyFileContents(fs, destFile, data);
+ }
+
+ /**
+ * Commit the file, with before and after checks on the dest and magic
+ * values.
+ * Failures can be set; they'll be reset after the commit.
+ * @param filename filename of file
+ * @param destFile destination path of file
+ * @param throttle probability of commit throttling
+ * @param failures failure limit
+ * @throws Exception any failure of the operation
+ */
+ private void commit(String filename,
+ Path destFile,
+ float throttle,
+ int failures) throws Exception {
+ commit(filename, makeMagic(destFile), destFile, throttle, failures);
+ }
+
+ /**
+ * Commit to a write to {@code magicFile} which is expected to
+ * be saved to {@code destFile}.
+ * Failures can be set; they'll be reset after the commit.
+ * @param magicFile path to write to
+ * @param destFile destination to verify
+ * @param throttle probability of commit throttling
+ * @param failures failure limit
+ */
+ private void commit(String filename,
+ Path magicFile,
+ Path destFile,
+ float throttle, int failures)
+ throws IOException {
+ resetFailures();
+ validateIntermediateAndFinalPaths(magicFile, destFile);
+ SinglePendingCommit commit = SinglePendingCommit.load(getFileSystem(),
+ validatePendingCommitData(filename, magicFile));
+ CommitOperations actions = newCommitOperations();
+ setThrottling(throttle, failures);
+ actions.commitOrFail(commit);
+ resetFailures();
+ verifyCommitExists(commit);
+ }
+
+ /**
+ * Perform any validation of paths.
+ * @param magicFilePath path to magic file
+ * @param destFile ultimate destination file
+ * @throws IOException IO failure
+ */
+ private void validateIntermediateAndFinalPaths(Path magicFilePath,
+ Path destFile)
+ throws IOException {
+ assertPathDoesNotExist("dest file was created", destFile);
+ }
+
+ /**
+ * Verify that the path at the end of a commit exists.
+ * This does not validate the size.
+ * @param commit commit to verify
+ * @throws FileNotFoundException dest doesn't exist
+ * @throws ValidationFailure commit arg is invalid
+ * @throws IOException invalid commit, IO failure
+ */
+ private void verifyCommitExists(SinglePendingCommit commit)
+ throws FileNotFoundException, ValidationFailure, IOException {
+ commit.validate();
+ // this will force an existence check
+ Path path = getFileSystem().keyToQualifiedPath(commit.getDestinationKey());
+ FileStatus status = getFileSystem().getFileStatus(path);
+ LOG.debug("Destination entry: {}", status);
+ if (!status.isFile()) {
+ throw new PathCommitException(path, "Not a file: " + status);
+ }
+ }
+
+ /**
+ * Validate that a pending commit data file exists, load it and validate
+ * its contents.
+ * @param filename short file name
+ * @param magicFile path that the file thinks that it was written to
+ * @return the path to the pending set
+ * @throws IOException IO problems
+ */
+ private Path validatePendingCommitData(String filename,
+ Path magicFile) throws IOException {
+ S3AFileSystem fs = getFileSystem();
+ Path pendingDataPath = new Path(magicFile.getParent(),
+ filename + PENDING_SUFFIX);
+ FileStatus fileStatus = verifyPathExists(fs, "no pending file",
+ pendingDataPath);
+ assertTrue("No data in " + fileStatus, fileStatus.getLen() > 0);
+ String data = read(fs, pendingDataPath);
+ LOG.info("Contents of {}: \n{}", pendingDataPath, data);
+ // really read it in and parse
+ SinglePendingCommit persisted = SinglePendingCommit.serializer()
+ .load(fs, pendingDataPath);
+ persisted.validate();
+ assertTrue("created timestamp wrong in " + persisted,
+ persisted.getCreated() > 0);
+ assertTrue("saved timestamp wrong in " + persisted,
+ persisted.getSaved() > 0);
+ List<String> etags = persisted.getEtags();
+ assertEquals("etag list " + persisted, 1, etags.size());
+ List<PartETag> partList = CommitOperations.toPartEtags(etags);
+ assertEquals("part list " + persisted, 1, partList.size());
+ return pendingDataPath;
+ }
+
+ /**
+ * Get a method-relative path.
+ * @param filename filename
+ * @return new path
+ * @throws IOException failure to create/parse the path.
+ */
+ private Path methodPath(String filename) throws IOException {
+ return new Path(methodPath(), filename);
+ }
+
+ /**
+ * Get a unique path for a method.
+ * @return a path
+ * @throws IOException
+ */
+ protected Path methodPath() throws IOException {
+ return path(getMethodName());
+ }
+
+ @Test
+ public void testUploadEmptyFile() throws Throwable {
+ File tempFile = File.createTempFile("commit", ".txt");
+ CommitOperations actions = newCommitOperations();
+ Path dest = methodPath("testUploadEmptyFile");
+ S3AFileSystem fs = getFileSystem();
+ fs.delete(dest, false);
+ fullThrottle();
+
+ SinglePendingCommit pendingCommit =
+ actions.uploadFileToPendingCommit(tempFile,
+ dest, null,
+ DEFAULT_MULTIPART_SIZE);
+ resetFailures();
+ assertPathDoesNotExist("pending commit", dest);
+ fullThrottle();
+ actions.commitOrFail(pendingCommit);
+ resetFailures();
+ FileStatus status = verifyPathExists(fs,
+ "uploaded file commit", dest);
+ assertEquals("File length in " + status, 0, status.getLen());
+ }
+
+ @Test
+ public void testUploadSmallFile() throws Throwable {
+ File tempFile = File.createTempFile("commit", ".txt");
+ String text = "hello, world";
+ FileUtils.write(tempFile, text, "UTF-8");
+ CommitOperations actions = newCommitOperations();
+ Path dest = methodPath("testUploadSmallFile");
+ S3AFileSystem fs = getFileSystem();
+ fullThrottle();
+ SinglePendingCommit pendingCommit =
+ actions.uploadFileToPendingCommit(tempFile,
+ dest, null,
+ DEFAULT_MULTIPART_SIZE);
+ resetFailures();
+ assertPathDoesNotExist("pending commit", dest);
+ fullThrottle();
+ actions.commitOrFail(pendingCommit);
+ resetFailures();
+ String s = readUTF8(fs, dest, -1);
+ assertEquals(text, s);
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testUploadMissingFile() throws Throwable {
+ File tempFile = File.createTempFile("commit", ".txt");
+ tempFile.delete();
+ CommitOperations actions = newCommitOperations();
+ Path dest = methodPath("testUploadMissingile");
+ fullThrottle();
+ actions.uploadFileToPendingCommit(tempFile, dest, null,
+ DEFAULT_MULTIPART_SIZE);
+ }
+
+ @Test
+ public void testRevertCommit() throws Throwable {
+ Path destFile = methodPath("part-0000");
+ S3AFileSystem fs = getFileSystem();
+ touch(fs, destFile);
+ CommitOperations actions = newCommitOperations();
+ SinglePendingCommit commit = new SinglePendingCommit();
+ commit.setDestinationKey(fs.pathToKey(destFile));
+ fullThrottle();
+ actions.revertCommit(commit);
+ resetFailures();
+ assertPathExists("parent of reverted commit", destFile.getParent());
+ }
+
+ @Test
+ public void testRevertMissingCommit() throws Throwable {
+ Path destFile = methodPath("part-0000");
+ S3AFileSystem fs = getFileSystem();
+ fs.delete(destFile, false);
+ CommitOperations actions = newCommitOperations();
+ SinglePendingCommit commit = new SinglePendingCommit();
+ commit.setDestinationKey(fs.pathToKey(destFile));
+ fullThrottle();
+ actions.revertCommit(commit);
+ assertPathExists("parent of reverted (nonexistent) commit",
+ destFile.getParent());
+ }
+
+ @Test
+ public void testFailuresInAbortListing() throws Throwable {
+ CommitOperations actions = newCommitOperations();
+ Path path = path("testFailuresInAbort");
+ getFileSystem().mkdirs(path);
+ setThrottling(HIGH_THROTTLE);
+ LOG.info("Aborting");
+ actions.abortPendingUploadsUnderPath(path);
+ LOG.info("Abort completed");
+ resetFailures();
+ }
+
+
+ /**
+ * Test a normal stream still works as expected in a magic filesystem,
+ * with a call of {@code hasCapability()} to check that it is normal.
+ * @throws Throwable failure
+ */
+ @Test
+ public void testWriteNormalStream() throws Throwable {
+ S3AFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT));
+
+ Path destFile = path("normal");
+ try (FSDataOutputStream out = fs.create(destFile, true)) {
+ out.writeChars("data");
+ assertFalse("stream has magic output: " + out,
+ out.hasCapability(STREAM_CAPABILITY_MAGIC_OUTPUT));
+ out.close();
+ }
+ FileStatus status = getFileStatusEventually(fs, destFile,
+ CONSISTENCY_WAIT);
+ assertTrue("Empty marker file: " + status, status.getLen() > 0);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/LoggingTextOutputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/LoggingTextOutputFormat.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/LoggingTextOutputFormat.java
new file mode 100644
index 0000000..1ac8038
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/LoggingTextOutputFormat.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
+import org.apache.hadoop.util.ReflectionUtils;
+
+/**
+ * A subclass of {@link TextOutputFormat} which logs what is happening, and
+ * returns a {@link LoggingLineRecordWriter} which allows the caller
+ * to get the destination path.
+ * @param <K> key
+ * @param <V> value
+ */
+public class LoggingTextOutputFormat<K, V> extends TextOutputFormat<K, V> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LoggingTextOutputFormat.class);
+
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.commit.LoggingTextOutputFormat";
+
+ @Override
+ public LoggingLineRecordWriter<K, V> getRecordWriter(TaskAttemptContext job)
+ throws IOException, InterruptedException {
+ Configuration conf = job.getConfiguration();
+ boolean isCompressed = getCompressOutput(job);
+ String keyValueSeparator = conf.get(SEPARATOR, "\t");
+ CompressionCodec codec = null;
+ String extension = "";
+ if (isCompressed) {
+ Class<? extends CompressionCodec> codecClass =
+ getOutputCompressorClass(job, GzipCodec.class);
+ codec = ReflectionUtils.newInstance(codecClass, conf);
+ extension = codec.getDefaultExtension();
+ }
+ Path file = getDefaultWorkFile(job, extension);
+ FileSystem fs = file.getFileSystem(conf);
+ FSDataOutputStream fileOut = fs.create(file, false);
+ LOG.debug("Creating LineRecordWriter with destination {}", file);
+ if (isCompressed) {
+ return new LoggingLineRecordWriter<>(
+ file, new DataOutputStream(codec.createOutputStream(fileOut)),
+ keyValueSeparator);
+ } else {
+ return new LoggingLineRecordWriter<>(file, fileOut, keyValueSeparator);
+ }
+ }
+
+ /**
+ * Write a line; counts the number of lines written and logs @ debug in the
+ * {@code close()} call.
+ * @param <K> key
+ * @param <V> value
+ */
+ public static class LoggingLineRecordWriter<K, V>
+ extends LineRecordWriter<K, V> {
+ private final Path dest;
+ private long lines;
+
+ public LoggingLineRecordWriter(Path dest, DataOutputStream out,
+ String keyValueSeparator) {
+ super(out, keyValueSeparator);
+ this.dest = dest;
+ }
+
+ public LoggingLineRecordWriter(DataOutputStream out, Path dest) {
+ super(out);
+ this.dest = dest;
+ }
+
+ @Override
+ public synchronized void write(K key, V value) throws IOException {
+ super.write(key, value);
+ lines++;
+ }
+
+ public synchronized void close(TaskAttemptContext context)
+ throws IOException {
+ LOG.debug("Closing output file {} with {} lines :{}",
+ dest, lines, out);
+ out.close();
+ }
+
+ public Path getDest() {
+ return dest;
+ }
+
+ public long getLines() {
+ return lines;
+ }
+ }
+
+ /**
+ * Bind to a configuration for job submission.
+ * @param conf configuration
+ */
+ public static void bind(Configuration conf) {
+ conf.setClass(MRJobConfig.OUTPUT_FORMAT_CLASS_ATTR,
+ LoggingTextOutputFormat.class,
+ OutputFormat.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java
new file mode 100644
index 0000000..7f689e0
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/MiniDFSClusterService.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.service.AbstractService;
+
+/**
+ * MiniDFS Cluster, encapsulated for use in different test suites.
+ */
+public class MiniDFSClusterService extends AbstractService {
+
+ public MiniDFSClusterService() {
+ super("MiniDFSTestCluster");
+ }
+
+ private MiniDFSCluster cluster = null;
+ private FileSystem clusterFS = null;
+ private LocalFileSystem localFS = null;
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ conf.setBoolean("dfs.webhdfs.enabled", false);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+ cluster = new MiniDFSCluster.Builder(conf)
+ .numDataNodes(1)
+ .format(true)
+ .racks(null)
+ .build();
+ clusterFS = cluster.getFileSystem();
+ localFS = FileSystem.getLocal(clusterFS.getConf());
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ clusterFS = null;
+ localFS = null;
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+
+ public MiniDFSCluster getCluster() {
+ return cluster;
+ }
+
+ public FileSystem getClusterFS() {
+ return clusterFS;
+ }
+
+ public LocalFileSystem getLocalFS() {
+ return localFS;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/49] hadoop git commit: YARN-6471. Support to add min/max resource
configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
index e3c108a..b9d5b82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableTable;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.junit.Assert;
@@ -52,6 +53,8 @@ public class TestPriorityUtilizationQueueOrderingPolicy {
when(q.getQueueCapacities()).thenReturn(qc);
when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i]));
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
list.add(q);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 1108f1a..0132348 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -354,10 +354,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
private void verifySubQueue(JSONObject info, String q,
float parentAbsCapacity, float parentAbsMaxCapacity)
throws JSONException, Exception {
- int numExpectedElements = 18;
+ int numExpectedElements = 20;
boolean isParentQueue = true;
if (!info.has("queues")) {
- numExpectedElements = 31;
+ numExpectedElements = 33;
isParentQueue = false;
}
assertEquals("incorrect number of elements", numExpectedElements, info.length());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/49] hadoop git commit: HDFS-12730. Verify open files captured in
the snapshots across config disable and enable.
Posted by su...@apache.org.
HDFS-12730. Verify open files captured in the snapshots across config disable and enable.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9fb4effd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9fb4effd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9fb4effd
Branch: refs/heads/YARN-5881
Commit: 9fb4effd2c4de2d83b667a43e8798315e85ff79b
Parents: 6f9d7a1
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Mon Nov 20 10:28:12 2017 -0800
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Mon Nov 20 10:28:12 2017 -0800
----------------------------------------------------------------------
.../namenode/snapshot/SnapshotManager.java | 8 +-
.../snapshot/TestOpenFilesWithSnapshot.java | 117 +++++++++++++++++++
2 files changed, 124 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fb4effd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
index 89033ef..58a218e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
@@ -35,6 +35,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import javax.management.ObjectName;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -75,7 +76,7 @@ public class SnapshotManager implements SnapshotStatsMXBean {
public static final Log LOG = LogFactory.getLog(SnapshotManager.class);
private final FSDirectory fsdir;
- private final boolean captureOpenFiles;
+ private boolean captureOpenFiles;
/**
* If skipCaptureAccessTimeOnlyChange is set to true, if accessTime
* of a file changed but there is no other modification made to the file,
@@ -121,6 +122,11 @@ public class SnapshotManager implements SnapshotStatsMXBean {
+ snapshotDiffAllowSnapRootDescendant);
}
+ @VisibleForTesting
+ void setCaptureOpenFiles(boolean captureOpenFiles) {
+ this.captureOpenFiles = captureOpenFiles;
+ }
+
/**
* @return skipCaptureAccessTimeOnlyChange
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9fb4effd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index 537612c..be118a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -843,6 +843,123 @@ public class TestOpenFilesWithSnapshot {
hbaseFileCksumBeforeTruncate, hbaseFileCksumS3);
}
+ private Path createSnapshot(Path snapRootDir, String snapName,
+ String fileName) throws Exception {
+ final Path snap1Dir = SnapshotTestHelper.createSnapshot(
+ fs, snapRootDir, snapName);
+ return new Path(snap1Dir, fileName);
+ }
+
+ private void verifyFileSize(long fileSize, Path... filePaths) throws
+ IOException {
+ for (Path filePath : filePaths) {
+ Assert.assertEquals(fileSize, fs.getFileStatus(filePath).getLen());
+ }
+ }
+
+ /**
+ * Verify open files captured in the snapshots across config disable
+ * and enable.
+ */
+ @Test
+ public void testOpenFilesWithMixedConfig() throws Exception {
+ final Path snapRootDir = new Path("/level_0_A");
+ final String flumeFileName = "flume.log";
+ final String snap1Name = "s1";
+ final String snap2Name = "s2";
+ final String snap3Name = "s3";
+ final String snap4Name = "s4";
+ final String snap5Name = "s5";
+
+ // Create files and open streams
+ final Path flumeFile = new Path(snapRootDir, flumeFileName);
+ createFile(flumeFile);
+ FSDataOutputStream flumeOutputStream = fs.append(flumeFile);
+
+ // 1. Disable capture open files
+ cluster.getNameNode().getNamesystem()
+ .getSnapshotManager().setCaptureOpenFiles(false);
+
+ // Create Snapshot S1
+ final Path flumeS1Path = createSnapshot(snapRootDir,
+ snap1Name, flumeFileName);
+
+ // Verify if Snap S1 file length is same as the the current versions
+ verifyFileSize(FILELEN, flumeS1Path);
+
+ // Write more data to files
+ long flumeFileWrittenDataLength = FILELEN;
+ int newWriteLength = (int) (BLOCKSIZE * 1.5);
+ byte[] buf = new byte[newWriteLength];
+ Random random = new Random();
+ random.nextBytes(buf);
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+
+ // Create Snapshot S2
+ final Path flumeS2Path = createSnapshot(snapRootDir,
+ snap2Name, flumeFileName);
+
+ // Since capture open files was disabled, all snapshots paths
+ // and the current version should have same file lengths.
+ verifyFileSize(flumeFileWrittenDataLength,
+ flumeFile, flumeS2Path, flumeS1Path);
+
+ // 2. Enable capture open files
+ cluster.getNameNode().getNamesystem()
+ .getSnapshotManager() .setCaptureOpenFiles(true);
+
+ // Write more data to files
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+ long flumeFileLengthAfterS3 = flumeFileWrittenDataLength;
+
+ // Create Snapshot S3
+ final Path flumeS3Path = createSnapshot(snapRootDir,
+ snap3Name, flumeFileName);
+
+ // Since open files captured in the previous snapshots were with config
+ // disabled, their file lengths are now same as the current version.
+ // With the config turned on, any new data written to the open files
+ // will no more reflect in the current version or old snapshot paths.
+ verifyFileSize(flumeFileWrittenDataLength, flumeFile, flumeS3Path,
+ flumeS2Path, flumeS1Path);
+
+ // Write more data to files
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+
+ // Create Snapshot S4
+ final Path flumeS4Path = createSnapshot(snapRootDir,
+ snap4Name, flumeFileName);
+
+ // Verify S4 has the latest data
+ verifyFileSize(flumeFileWrittenDataLength, flumeFile, flumeS4Path);
+
+ // But, open files captured as of Snapshot S3 and before should
+ // have their old file lengths intact.
+ verifyFileSize(flumeFileLengthAfterS3, flumeS3Path,
+ flumeS2Path, flumeS1Path);
+
+ long flumeFileLengthAfterS4 = flumeFileWrittenDataLength;
+
+ // 3. Disable capture open files
+ cluster.getNameNode().getNamesystem()
+ .getSnapshotManager() .setCaptureOpenFiles(false);
+
+ // Create Snapshot S5
+ final Path flumeS5Path = createSnapshot(snapRootDir,
+ snap5Name, flumeFileName);
+
+ flumeFileWrittenDataLength += writeToStream(flumeOutputStream, buf);
+
+ // Since capture open files was disabled, any snapshots taken after the
+ // config change and the current version should have same file lengths
+ // for the open files.
+ verifyFileSize(flumeFileWrittenDataLength, flumeFile, flumeS5Path);
+
+ // But, the old snapshots taken before the config disable should
+ // continue to be consistent.
+ verifyFileSize(flumeFileLengthAfterS4, flumeS4Path);
+ }
+
private void restartNameNode() throws Exception {
cluster.triggerBlockReports();
NameNode nameNode = cluster.getNameNode();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/49] hadoop git commit: HADOOP-13514. Upgrade maven surefire
plugin to 2.20.1
Posted by su...@apache.org.
HADOOP-13514. Upgrade maven surefire plugin to 2.20.1
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6903cf09
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6903cf09
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6903cf09
Branch: refs/heads/YARN-5881
Commit: 6903cf096e702c8d4827d4e72cd5638c3a3b5429
Parents: b5b81a4
Author: Akira Ajisaka <aa...@apache.org>
Authored: Sun Nov 19 12:39:02 2017 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Sun Nov 19 12:39:37 2017 -0800
----------------------------------------------------------------------
BUILDING.txt | 4 ++--
hadoop-client-modules/hadoop-client-integration-tests/pom.xml | 3 +++
hadoop-project/pom.xml | 5 +++--
hadoop-tools/hadoop-aws/pom.xml | 2 ++
hadoop-tools/hadoop-azure/pom.xml | 3 +++
5 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6903cf09/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 47aaab4..9955563 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -331,10 +331,10 @@ If the build process fails with an out of memory error, you should be able to fi
it by increasing the memory used by maven which can be done via the environment
variable MAVEN_OPTS.
-Here is an example setting to allocate between 256 and 512 MB of heap space to
+Here is an example setting to allocate between 256 MB and 1 GB of heap space to
Maven
-export MAVEN_OPTS="-Xms256m -Xmx512m"
+export MAVEN_OPTS="-Xms256m -Xmx1g"
----------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6903cf09/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
index 1295b59..89b9645 100644
--- a/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
+++ b/hadoop-client-modules/hadoop-client-integration-tests/pom.xml
@@ -128,6 +128,9 @@
<goal>integration-test</goal>
<goal>verify</goal>
</goals>
+ <configuration>
+ <trimStackTrace>false</trimStackTrace>
+ </configuration>
</execution>
</executions>
</plugin>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6903cf09/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index f99e413..c4dc1bf 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -118,7 +118,7 @@
<!-- Plugin versions and config -->
<maven-surefire-plugin.argLine>-Xmx2048m -XX:+HeapDumpOnOutOfMemoryError</maven-surefire-plugin.argLine>
- <maven-surefire-plugin.version>2.17</maven-surefire-plugin.version>
+ <maven-surefire-plugin.version>2.20.1</maven-surefire-plugin.version>
<maven-surefire-report-plugin.version>${maven-surefire-plugin.version}</maven-surefire-report-plugin.version>
<maven-failsafe-plugin.version>${maven-surefire-plugin.version}</maven-failsafe-plugin.version>
@@ -1602,6 +1602,7 @@
<DYLD_LIBRARY_PATH>${env.DYLD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib</DYLD_LIBRARY_PATH>
<MALLOC_ARENA_MAX>4</MALLOC_ARENA_MAX>
</environmentVariables>
+ <trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<hadoop.log.dir>${project.build.directory}/log</hadoop.log.dir>
@@ -1612,7 +1613,7 @@
<test.build.data>${test.build.data}</test.build.data>
<test.build.webapps>${test.build.webapps}</test.build.webapps>
<test.cache.data>${test.cache.data}</test.cache.data>
- <test.build.classes>${test.build.classes}</test.build.classes>
+ <test.build.classes>${project.build.directory}/test-classes</test.build.classes>
<java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
<java.security.krb5.conf>${project.build.directory}/test-classes/krb5.conf</java.security.krb5.conf>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6903cf09/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 47788cd..97ceddf 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -153,6 +153,7 @@
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<!-- Tell tests that they are being executed in parallel -->
<test.parallel.execution>true</test.parallel.execution>
@@ -207,6 +208,7 @@
</goals>
<configuration>
<forkedProcessTimeoutInSeconds>${fs.s3a.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<!-- Tell tests that they are being executed sequentially -->
<test.parallel.execution>false</test.parallel.execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6903cf09/hadoop-tools/hadoop-azure/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/pom.xml b/hadoop-tools/hadoop-azure/pom.xml
index b479872..1728b60 100644
--- a/hadoop-tools/hadoop-azure/pom.xml
+++ b/hadoop-tools/hadoop-azure/pom.xml
@@ -349,6 +349,7 @@
<reuseForks>false</reuseForks>
<argLine>${maven-surefire-plugin.argLine} -DminiClusterDedicatedDirs=true</argLine>
<forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<!-- Tell tests that they are being executed in parallel -->
<test.parallel.execution>true</test.parallel.execution>
@@ -404,6 +405,7 @@
</goals>
<configuration>
<forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
<systemPropertyVariables>
<test.parallel.execution>false</test.parallel.execution>
<fs.azure.scale.test.enabled>${fs.azure.scale.test.enabled}</fs.azure.scale.test.enabled>
@@ -454,6 +456,7 @@
<fs.azure.scale.test.timeout>${fs.azure.scale.test.timeout}</fs.azure.scale.test.timeout>
</systemPropertyVariables>
<forkedProcessTimeoutInSeconds>${fs.azure.scale.test.timeout}</forkedProcessTimeoutInSeconds>
+ <trimStackTrace>false</trimStackTrace>
</configuration>
</execution>
</executions>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/49] hadoop git commit: YARN-6128. Add support for AMRMProxy HA.
(Botong Huang via Subru).
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
index d63b2cf..ebd85bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyService.java
@@ -34,12 +34,13 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SaslRpcServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
@@ -60,15 +61,19 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.ContainerType;
+import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
import org.apache.hadoop.yarn.server.nodemanager.scheduler.DistributedScheduler;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -82,7 +87,7 @@ import com.google.common.base.Preconditions;
* pipeline is a chain of interceptor instances that can inspect and modify the
* request/response as needed.
*/
-public class AMRMProxyService extends AbstractService implements
+public class AMRMProxyService extends CompositeService implements
ApplicationMasterProtocol {
private static final Logger LOG = LoggerFactory
.getLogger(AMRMProxyService.class);
@@ -96,6 +101,7 @@ public class AMRMProxyService extends AbstractService implements
private InetSocketAddress listenerEndpoint;
private AMRMProxyTokenSecretManager secretManager;
private Map<ApplicationId, RequestInterceptorChainWrapper> applPipelineMap;
+ private RegistryOperations registry;
/**
* Creates an instance of the service.
@@ -118,10 +124,23 @@ public class AMRMProxyService extends AbstractService implements
@Override
protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
this.secretManager =
new AMRMProxyTokenSecretManager(this.nmContext.getNMStateStore());
this.secretManager.init(conf);
+
+ // Both second app attempt and NM restart within Federation need registry
+ if (conf.getBoolean(YarnConfiguration.AMRM_PROXY_HA_ENABLED,
+ YarnConfiguration.DEFAULT_AMRM_PROXY_HA_ENABLED)
+ || conf.getBoolean(YarnConfiguration.NM_RECOVERY_ENABLED,
+ YarnConfiguration.DEFAULT_NM_RECOVERY_ENABLED)) {
+ this.registry = FederationStateStoreFacade.createInstance(conf,
+ YarnConfiguration.YARN_REGISTRY_CLASS,
+ YarnConfiguration.DEFAULT_YARN_REGISTRY_CLASS,
+ RegistryOperations.class);
+ addService(this.registry);
+ }
+
+ super.serviceInit(conf);
}
@Override
@@ -203,6 +222,8 @@ public class AMRMProxyService extends AbstractService implements
amrmToken = new Token<>();
amrmToken.decodeFromUrlString(
new String(contextEntry.getValue(), "UTF-8"));
+ // Clear the service field, as if RM just issued the token
+ amrmToken.setService(new Text());
}
}
@@ -214,12 +235,36 @@ public class AMRMProxyService extends AbstractService implements
throw new IOException("No user found for app attempt " + attemptId);
}
+ // Regenerate the local AMRMToken for the AM
Token<AMRMTokenIdentifier> localToken =
this.secretManager.createAndGetAMRMToken(attemptId);
+ // Retrieve the AM container credentials from NM context
+ Credentials amCred = null;
+ for (Container container : this.nmContext.getContainers().values()) {
+ LOG.debug("From NM Context container " + container.getContainerId());
+ if (container.getContainerId().getApplicationAttemptId().equals(
+ attemptId) && container.getContainerTokenIdentifier() != null) {
+ LOG.debug("Container type "
+ + container.getContainerTokenIdentifier().getContainerType());
+ if (container.getContainerTokenIdentifier()
+ .getContainerType() == ContainerType.APPLICATION_MASTER) {
+ LOG.info("AM container {} found in context, has credentials: {}",
+ container.getContainerId(),
+ (container.getCredentials() != null));
+ amCred = container.getCredentials();
+ }
+ }
+ }
+ if (amCred == null) {
+ LOG.error("No credentials found for AM container of {}. "
+ + "Yarn registry access might not work", attemptId);
+ }
+
+ // Create the intercepter pipeline for the AM
initializePipeline(attemptId, user, amrmToken, localToken,
- entry.getValue(), true);
- } catch (Exception e) {
+ entry.getValue(), true, amCred);
+ } catch (IOException e) {
LOG.error("Exception when recovering " + attemptId
+ ", removing it from NMStateStore and move on", e);
this.nmContext.getNMStateStore().removeAMRMProxyAppContext(attemptId);
@@ -326,7 +371,7 @@ public class AMRMProxyService extends AbstractService implements
initializePipeline(appAttemptId,
containerTokenIdentifierForKey.getApplicationSubmitter(), amrmToken,
- localToken, null, false);
+ localToken, null, false, credentials);
}
/**
@@ -342,7 +387,8 @@ public class AMRMProxyService extends AbstractService implements
protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
String user, Token<AMRMTokenIdentifier> amrmToken,
Token<AMRMTokenIdentifier> localToken,
- Map<String, byte[]> recoveredDataMap, boolean isRecovery) {
+ Map<String, byte[]> recoveredDataMap, boolean isRecovery,
+ Credentials credentials) {
RequestInterceptorChainWrapper chainWrapper = null;
synchronized (applPipelineMap) {
if (applPipelineMap
@@ -404,8 +450,9 @@ public class AMRMProxyService extends AbstractService implements
try {
RequestInterceptor interceptorChain =
this.createRequestInterceptorChain();
- interceptorChain.init(createApplicationMasterContext(this.nmContext,
- applicationAttemptId, user, amrmToken, localToken));
+ interceptorChain.init(
+ createApplicationMasterContext(this.nmContext, applicationAttemptId,
+ user, amrmToken, localToken, credentials, this.registry));
if (isRecovery) {
if (recoveredDataMap == null) {
throw new YarnRuntimeException(
@@ -497,14 +544,12 @@ public class AMRMProxyService extends AbstractService implements
allocateResponse.setAMRMToken(null);
org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> newToken =
- new org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>(
- token.getIdentifier().array(), token.getPassword().array(),
- new Text(token.getKind()), new Text(token.getService()));
-
- context.setAMRMToken(newToken);
+ ConverterUtils.convertFromYarn(token, (Text) null);
- // Update the AMRMToken in context map in NM state store
- if (this.nmContext.getNMStateStore() != null) {
+ // Update the AMRMToken in context map, and in NM state store if it is
+ // different
+ if (context.setAMRMToken(newToken)
+ && this.nmContext.getNMStateStore() != null) {
try {
this.nmContext.getNMStateStore().storeAMRMProxyAppContextEntry(
context.getApplicationAttemptId(), NMSS_AMRMTOKEN_KEY,
@@ -547,10 +592,12 @@ public class AMRMProxyService extends AbstractService implements
private AMRMProxyApplicationContext createApplicationMasterContext(
Context context, ApplicationAttemptId applicationAttemptId, String user,
Token<AMRMTokenIdentifier> amrmToken,
- Token<AMRMTokenIdentifier> localToken) {
+ Token<AMRMTokenIdentifier> localToken, Credentials credentials,
+ RegistryOperations registryImpl) {
AMRMProxyApplicationContextImpl appContext =
new AMRMProxyApplicationContextImpl(context, getConfig(),
- applicationAttemptId, user, amrmToken, localToken);
+ applicationAttemptId, user, amrmToken, localToken, credentials,
+ registryImpl);
return appContext;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
index 33cfca3..ef5e061 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/FederationInterceptor.java
@@ -34,6 +34,8 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -42,6 +44,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -56,17 +59,20 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidApplicationMasterRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProviderUtil;
import org.apache.hadoop.yarn.server.federation.policies.FederationPolicyUtils;
import org.apache.hadoop.yarn.server.federation.policies.amrmproxy.FederationAMRMProxyPolicy;
import org.apache.hadoop.yarn.server.federation.policies.exceptions.FederationPolicyInitializationException;
import org.apache.hadoop.yarn.server.federation.resolver.SubClusterResolver;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.utils.FederationRegistryClient;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.uam.UnmanagedAMPoolManager;
import org.apache.hadoop.yarn.server.utils.AMRMClientUtils;
import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import org.apache.hadoop.yarn.util.AsyncCallback;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -145,6 +151,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
*/
private UserGroupInformation appOwner;
+ private FederationRegistryClient registryClient;
+
/**
* Creates an instance of the FederationInterceptor class.
*/
@@ -179,6 +187,10 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
} catch (Exception ex) {
throw new YarnRuntimeException(ex);
}
+ // Add all app tokens for Yarn Registry access
+ if (this.registryClient != null && appContext.getCredentials() != null) {
+ this.appOwner.addCredentials(appContext.getCredentials());
+ }
this.homeSubClusterId =
SubClusterId.newInstance(YarnConfiguration.getClusterId(conf));
@@ -192,6 +204,11 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
this.uamPool.init(conf);
this.uamPool.start();
+
+ if (appContext.getRegistryClient() != null) {
+ this.registryClient = new FederationRegistryClient(conf,
+ appContext.getRegistryClient(), this.appOwner);
+ }
}
/**
@@ -250,20 +267,27 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
*/
this.amRegistrationResponse =
this.homeRM.registerApplicationMaster(request);
+ if (this.amRegistrationResponse
+ .getContainersFromPreviousAttempts() != null) {
+ cacheAllocatedContainers(
+ this.amRegistrationResponse.getContainersFromPreviousAttempts(),
+ this.homeSubClusterId);
+ }
+
+ ApplicationId appId =
+ getApplicationContext().getApplicationAttemptId().getApplicationId();
+ reAttachUAMAndMergeRegisterResponse(this.amRegistrationResponse, appId);
// the queue this application belongs will be used for getting
// AMRMProxy policy from state store.
String queue = this.amRegistrationResponse.getQueue();
if (queue == null) {
- LOG.warn("Received null queue for application "
- + getApplicationContext().getApplicationAttemptId().getApplicationId()
- + " from home sub-cluster. Will use default queue name "
+ LOG.warn("Received null queue for application " + appId
+ + " from home subcluster. Will use default queue name "
+ YarnConfiguration.DEFAULT_QUEUE_NAME
+ " for getting AMRMProxyPolicy");
} else {
- LOG.info("Application "
- + getApplicationContext().getApplicationAttemptId().getApplicationId()
- + " belongs to queue " + queue);
+ LOG.info("Application " + appId + " belongs to queue " + queue);
}
// Initialize the AMRMProxyPolicy
@@ -304,7 +328,7 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
AllocateResponse homeResponse = AMRMClientUtils.allocateWithReRegister(
requests.get(this.homeSubClusterId), this.homeRM,
this.amRegistrationRequest,
- getApplicationContext().getApplicationAttemptId());
+ getApplicationContext().getApplicationAttemptId().getApplicationId());
// Notify policy of home response
try {
@@ -393,8 +417,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
// request to the home resource manager on this thread.
FinishApplicationMasterResponse homeResponse =
AMRMClientUtils.finishAMWithReRegister(request, this.homeRM,
- this.amRegistrationRequest,
- getApplicationContext().getApplicationAttemptId());
+ this.amRegistrationRequest, getApplicationContext()
+ .getApplicationAttemptId().getApplicationId());
if (subClusterIds.size() > 0) {
// Wait for other sub-cluster resource managers to return the
@@ -425,6 +449,14 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
if (failedToUnRegister) {
homeResponse.setIsUnregistered(false);
+ } else {
+ // Clean up UAMs only when the app finishes successfully, so that no more
+ // attempt will be launched.
+ this.uamPool.stop();
+ if (this.registryClient != null) {
+ this.registryClient.removeAppFromRegistry(getApplicationContext()
+ .getApplicationAttemptId().getApplicationId());
+ }
}
return homeResponse;
}
@@ -442,9 +474,8 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
*/
@Override
public void shutdown() {
- if (this.uamPool != null) {
- this.uamPool.stop();
- }
+ // Do not stop uamPool service and kill UAMs here because of possible second
+ // app attempt
if (threadpool != null) {
try {
threadpool.shutdown();
@@ -456,6 +487,16 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
}
/**
+ * Only for unit test cleanup.
+ */
+ @VisibleForTesting
+ protected void cleanupRegistry() {
+ if (this.registryClient != null) {
+ this.registryClient.cleanAllApplications();
+ }
+ }
+
+ /**
* Create the UAM pool manager for secondary sub-clsuters. For unit test to
* override.
*
@@ -486,6 +527,120 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
}
}
+ private void mergeRegisterResponse(
+ RegisterApplicationMasterResponse homeResponse,
+ RegisterApplicationMasterResponse otherResponse) {
+
+ if (!isNullOrEmpty(otherResponse.getContainersFromPreviousAttempts())) {
+ if (!isNullOrEmpty(homeResponse.getContainersFromPreviousAttempts())) {
+ homeResponse.getContainersFromPreviousAttempts()
+ .addAll(otherResponse.getContainersFromPreviousAttempts());
+ } else {
+ homeResponse.setContainersFromPreviousAttempts(
+ otherResponse.getContainersFromPreviousAttempts());
+ }
+ }
+
+ if (!isNullOrEmpty(otherResponse.getNMTokensFromPreviousAttempts())) {
+ if (!isNullOrEmpty(homeResponse.getNMTokensFromPreviousAttempts())) {
+ homeResponse.getNMTokensFromPreviousAttempts()
+ .addAll(otherResponse.getNMTokensFromPreviousAttempts());
+ } else {
+ homeResponse.setNMTokensFromPreviousAttempts(
+ otherResponse.getNMTokensFromPreviousAttempts());
+ }
+ }
+ }
+
+ /**
+ * Try re-attach to all existing and running UAMs in secondary sub-clusters
+ * launched by previous application attempts if any. All running containers in
+ * the UAMs will be combined into the registerResponse. For the first attempt,
+ * the registry will be empty for this application and thus no-op here.
+ */
+ protected void reAttachUAMAndMergeRegisterResponse(
+ RegisterApplicationMasterResponse homeResponse,
+ final ApplicationId appId) {
+
+ if (this.registryClient == null) {
+ // Both AMRMProxy HA and NM work preserving restart is not enabled
+ LOG.warn("registryClient is null, skip attaching existing UAM if any");
+ return;
+ }
+
+ // Load existing running UAMs from the previous attempts from
+ // registry, if any
+ Map<String, Token<AMRMTokenIdentifier>> uamMap =
+ this.registryClient.loadStateFromRegistry(appId);
+ if (uamMap.size() == 0) {
+ LOG.info("No existing UAM for application {} found in Yarn Registry",
+ appId);
+ return;
+ }
+ LOG.info("Found {} existing UAMs for application {} in Yarn Registry. "
+ + "Reattaching in parallel", uamMap.size(), appId);
+
+ ExecutorCompletionService<RegisterApplicationMasterResponse>
+ completionService = new ExecutorCompletionService<>(threadpool);
+
+ for (Entry<String, Token<AMRMTokenIdentifier>> entry : uamMap.entrySet()) {
+ final SubClusterId subClusterId =
+ SubClusterId.newInstance(entry.getKey());
+ final Token<AMRMTokenIdentifier> amrmToken = entry.getValue();
+
+ completionService
+ .submit(new Callable<RegisterApplicationMasterResponse>() {
+ @Override
+ public RegisterApplicationMasterResponse call() throws Exception {
+ RegisterApplicationMasterResponse response = null;
+ try {
+ // Create a config loaded with federation on and subclusterId
+ // for each UAM
+ YarnConfiguration config = new YarnConfiguration(getConf());
+ FederationProxyProviderUtil.updateConfForFederation(config,
+ subClusterId.getId());
+
+ uamPool.reAttachUAM(subClusterId.getId(), config, appId,
+ amRegistrationResponse.getQueue(),
+ getApplicationContext().getUser(), homeSubClusterId.getId(),
+ amrmToken);
+
+ response = uamPool.registerApplicationMaster(
+ subClusterId.getId(), amRegistrationRequest);
+
+ if (response != null
+ && response.getContainersFromPreviousAttempts() != null) {
+ cacheAllocatedContainers(
+ response.getContainersFromPreviousAttempts(),
+ subClusterId);
+ }
+ LOG.info("UAM {} reattached for {}", subClusterId, appId);
+ } catch (Throwable e) {
+ LOG.error(
+ "Reattaching UAM " + subClusterId + " failed for " + appId,
+ e);
+ }
+ return response;
+ }
+ });
+ }
+
+ // Wait for the re-attach responses
+ for (int i = 0; i < uamMap.size(); i++) {
+ try {
+ Future<RegisterApplicationMasterResponse> future =
+ completionService.take();
+ RegisterApplicationMasterResponse registerResponse = future.get();
+ if (registerResponse != null) {
+ LOG.info("Merging register response for {}", appId);
+ mergeRegisterResponse(homeResponse, registerResponse);
+ }
+ } catch (Exception e) {
+ LOG.warn("Reattaching UAM failed for ApplicationId: " + appId, e);
+ }
+ }
+ }
+
private SubClusterId getSubClusterForNode(String nodeName) {
SubClusterId subClusterId = null;
try {
@@ -655,6 +810,20 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
responses.add(response);
}
+ // Save the new AMRMToken for the UAM in registry if present
+ if (response.getAMRMToken() != null) {
+ Token<AMRMTokenIdentifier> newToken = ConverterUtils
+ .convertFromYarn(response.getAMRMToken(), (Text) null);
+ // Update the token in registry
+ if (registryClient != null) {
+ registryClient
+ .writeAMRMTokenForUAM(
+ getApplicationContext().getApplicationAttemptId()
+ .getApplicationId(),
+ subClusterId.getId(), newToken);
+ }
+ }
+
// Notify policy of secondary sub-cluster responses
try {
policyInterpreter.notifyOfResponse(subClusterId, response);
@@ -714,20 +883,23 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
subClusterId);
RegisterApplicationMasterResponse uamResponse = null;
+ Token<AMRMTokenIdentifier> token = null;
try {
// For appNameSuffix, use subClusterId of the home sub-cluster
- uamResponse = uamPool.createAndRegisterNewUAM(subClusterId,
- registerRequest, config,
+ token = uamPool.launchUAM(subClusterId, config,
appContext.getApplicationAttemptId().getApplicationId(),
amRegistrationResponse.getQueue(), appContext.getUser(),
- homeSubClusterId.toString());
+ homeSubClusterId.toString(), registryClient != null);
+
+ uamResponse = uamPool.registerApplicationMaster(subClusterId,
+ registerRequest);
} catch (Throwable e) {
LOG.error("Failed to register application master: "
+ subClusterId + " Application: "
+ appContext.getApplicationAttemptId(), e);
}
return new RegisterApplicationMasterResponseInfo(uamResponse,
- SubClusterId.newInstance(subClusterId));
+ SubClusterId.newInstance(subClusterId), token);
}
});
}
@@ -752,6 +924,14 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
+ getApplicationContext().getApplicationAttemptId());
successfulRegistrations.put(uamResponse.getSubClusterId(),
uamResponse.getResponse());
+
+ if (registryClient != null) {
+ registryClient.writeAMRMTokenForUAM(
+ getApplicationContext().getApplicationAttemptId()
+ .getApplicationId(),
+ uamResponse.getSubClusterId().getId(),
+ uamResponse.getUamToken());
+ }
}
} catch (Exception e) {
LOG.warn("Failed to register unmanaged application master: "
@@ -1087,11 +1267,14 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
private static class RegisterApplicationMasterResponseInfo {
private RegisterApplicationMasterResponse response;
private SubClusterId subClusterId;
+ private Token<AMRMTokenIdentifier> uamToken;
RegisterApplicationMasterResponseInfo(
- RegisterApplicationMasterResponse response, SubClusterId subClusterId) {
+ RegisterApplicationMasterResponse response, SubClusterId subClusterId,
+ Token<AMRMTokenIdentifier> uamToken) {
this.response = response;
this.subClusterId = subClusterId;
+ this.uamToken = uamToken;
}
public RegisterApplicationMasterResponse getResponse() {
@@ -1101,6 +1284,10 @@ public class FederationInterceptor extends AbstractRequestInterceptor {
public SubClusterId getSubClusterId() {
return subClusterId;
}
+
+ public Token<AMRMTokenIdentifier> getUamToken() {
+ return uamToken;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 831ba0b..44bfc68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -353,10 +353,6 @@ public class ContainerManagerImpl extends CompositeService implements
rsrcLocalizationSrvc.recoverLocalizedResources(
stateStore.loadLocalizationState());
- if (this.amrmProxyEnabled) {
- this.getAMRMProxyService().recover();
- }
-
RecoveredApplicationsState appsState = stateStore.loadApplicationsState();
for (ContainerManagerApplicationProto proto :
appsState.getApplications()) {
@@ -373,6 +369,11 @@ public class ContainerManagerImpl extends CompositeService implements
recoverContainer(rcs);
}
+ // Recovery AMRMProxy state after apps and containers are recovered
+ if (this.amrmProxyEnabled) {
+ this.getAMRMProxyService().recover();
+ }
+
//Dispatching the RECOVERY_COMPLETED event through the dispatcher
//so that all the paused, scheduled and queued containers will
//be scheduled for execution on availability of resources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index 3c57496..da1d047 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -56,10 +57,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
-import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerAllocator;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -179,6 +180,15 @@ public abstract class BaseAMRMProxyTest {
return new NMContext(null, null, null, null, stateStore, false, this.conf);
}
+ protected List<ContainerId> getCompletedContainerIds(
+ List<ContainerStatus> containerStatus) {
+ List<ContainerId> ret = new ArrayList<>();
+ for (ContainerStatus status : containerStatus) {
+ ret.add(status.getContainerId());
+ }
+ return ret;
+ }
+
/**
* This helper method will invoke the specified function in parallel for each
* end point in the specified list using a thread pool and return the
@@ -623,7 +633,7 @@ public abstract class BaseAMRMProxyTest {
*/
public void initApp(ApplicationAttemptId applicationId, String user) {
super.initializePipeline(applicationId, user,
- new Token<AMRMTokenIdentifier>(), null, null, false);
+ new Token<AMRMTokenIdentifier>(), null, null, false, null);
}
public void stopApp(ApplicationId applicationId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 937ede5..b955311 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -444,7 +444,7 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
applicationAttemptId = ApplicationAttemptId.newInstance(appId, 2);
getAMRMProxyService().initializePipeline(applicationAttemptId, user,
- new Token<AMRMTokenIdentifier>(), null, null, false);
+ new Token<AMRMTokenIdentifier>(), null, null, false, null);
RequestInterceptorChainWrapper chain2 =
getAMRMProxyService().getPipelines().get(appId);
@@ -531,16 +531,14 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
"new AMRMToken from RM should have been nulled by AMRMProxyService",
allocateResponse.getAMRMToken());
- // The way the mock resource manager is setup, it will return the containers
- // that were released in the response. This is done because the UAMs run
- // asynchronously and we need to if all the resource managers received the
- // release it. The containers sent by the mock resource managers will be
+ // We need to make sure all the resource managers received the
+ // release list. The containers sent by the mock resource managers will be
// aggregated and returned back to us and we can assert if all the release
// lists reached the sub-clusters
- List<Container> containersForReleasedContainerIds =
- new ArrayList<Container>();
- containersForReleasedContainerIds.addAll(allocateResponse
- .getAllocatedContainers());
+ List<ContainerId> containersForReleasedContainerIds = new ArrayList<>();
+ List<ContainerId> newlyFinished = getCompletedContainerIds(
+ allocateResponse.getCompletedContainersStatuses());
+ containersForReleasedContainerIds.addAll(newlyFinished);
// Send max 10 heart beats to receive all the containers. If not, we will
// fail the test
@@ -554,8 +552,9 @@ public class TestAMRMProxyService extends BaseAMRMProxyTest {
"new AMRMToken from RM should have been nulled by AMRMProxyService",
allocateResponse.getAMRMToken());
- containersForReleasedContainerIds.addAll(allocateResponse
- .getAllocatedContainers());
+ newlyFinished = getCompletedContainerIds(
+ allocateResponse.getCompletedContainersStatuses());
+ containersForReleasedContainerIds.addAll(newlyFinished);
LOG.info("Number of containers received in this request: "
+ Integer.toString(allocateResponse.getAllocatedContainers()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
index 3db0e35..aa7ed69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestFederationInterceptor.java
@@ -19,16 +19,20 @@
package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
@@ -59,6 +63,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -79,7 +87,10 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
private TestableFederationInterceptor interceptor;
private MemoryFederationStateStore stateStore;
+ private NMStateStoreService nmStateStore;
+ private RegistryOperations registry;
+ private Context nmContext;
private int testAppId;
private ApplicationAttemptId attemptId;
@@ -93,15 +104,28 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
FederationStateStoreFacade.getInstance().reinitialize(stateStore,
getConf());
+ nmStateStore = new NMMemoryStateStoreService();
+ nmStateStore.init(getConf());
+ nmStateStore.start();
+
+ registry = new FSRegistryOperationsService();
+ registry.init(getConf());
+ registry.start();
+
testAppId = 1;
attemptId = getApplicationAttemptId(testAppId);
- interceptor.init(new AMRMProxyApplicationContextImpl(null, getConf(),
- attemptId, "test-user", null, null));
+ nmContext =
+ new NMContext(null, null, null, null, nmStateStore, false, getConf());
+ interceptor.init(new AMRMProxyApplicationContextImpl(nmContext, getConf(),
+ attemptId, "test-user", null, null, null, registry));
+ interceptor.cleanupRegistry();
}
@Override
public void tearDown() {
+ interceptor.cleanupRegistry();
interceptor.shutdown();
+ registry.stop();
super.tearDown();
}
@@ -207,18 +231,17 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
AllocateResponse allocateResponse = interceptor.allocate(allocateRequest);
Assert.assertNotNull(allocateResponse);
- // The way the mock resource manager is setup, it will return the containers
- // that were released in the allocated containers. The release request will
- // be split and handled by the corresponding UAM. The release containers
- // returned by the mock resource managers will be aggregated and returned
- // back to us and we can check if total request size and returned size are
- // the same
- List<Container> containersForReleasedContainerIds =
- new ArrayList<Container>();
- containersForReleasedContainerIds
- .addAll(allocateResponse.getAllocatedContainers());
+ // The release request will be split and handled by the corresponding UAM.
+ // The release containers returned by the mock resource managers will be
+ // aggregated and returned back to us and we can check if total request size
+ // and returned size are the same
+ List<ContainerId> containersForReleasedContainerIds =
+ new ArrayList<ContainerId>();
+ List<ContainerId> newlyFinished = getCompletedContainerIds(
+ allocateResponse.getCompletedContainersStatuses());
+ containersForReleasedContainerIds.addAll(newlyFinished);
LOG.info("Number of containers received in the original request: "
- + Integer.toString(allocateResponse.getAllocatedContainers().size()));
+ + Integer.toString(newlyFinished.size()));
// Send max 10 heart beats to receive all the containers. If not, we will
// fail the test
@@ -228,11 +251,12 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
allocateResponse =
interceptor.allocate(Records.newRecord(AllocateRequest.class));
Assert.assertNotNull(allocateResponse);
- containersForReleasedContainerIds
- .addAll(allocateResponse.getAllocatedContainers());
+ newlyFinished = getCompletedContainerIds(
+ allocateResponse.getCompletedContainersStatuses());
+ containersForReleasedContainerIds.addAll(newlyFinished);
LOG.info("Number of containers received in this request: "
- + Integer.toString(allocateResponse.getAllocatedContainers().size()));
+ + Integer.toString(newlyFinished.size()));
LOG.info("Total number of containers received: "
+ Integer.toString(containersForReleasedContainerIds.size()));
Thread.sleep(10);
@@ -547,4 +571,74 @@ public class TestFederationInterceptor extends BaseAMRMProxyTest {
Assert.assertEquals(1, response.getUpdatedContainers().size());
Assert.assertEquals(1, response.getUpdateErrors().size());
}
+
+ @Test
+ public void testSecondAttempt() throws Exception {
+ ApplicationUserInfo userInfo = getApplicationUserInfo(testAppId);
+ userInfo.getUser().doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ // Register the application
+ RegisterApplicationMasterRequest registerReq =
+ Records.newRecord(RegisterApplicationMasterRequest.class);
+ registerReq.setHost(Integer.toString(testAppId));
+ registerReq.setRpcPort(testAppId);
+ registerReq.setTrackingUrl("");
+
+ RegisterApplicationMasterResponse registerResponse =
+ interceptor.registerApplicationMaster(registerReq);
+ Assert.assertNotNull(registerResponse);
+
+ Assert.assertEquals(0, interceptor.getUnmanagedAMPoolSize());
+
+ // Allocate one batch of containers
+ registerSubCluster(SubClusterId.newInstance("SC-1"));
+ registerSubCluster(SubClusterId.newInstance(HOME_SC_ID));
+
+ int numberOfContainers = 3;
+ List<Container> containers =
+ getContainersAndAssert(numberOfContainers, numberOfContainers * 2);
+ for (Container c : containers) {
+ System.out.println(c.getId() + " ha");
+ }
+ Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+
+ // Preserve the mock RM instances for secondaries
+ ConcurrentHashMap<String, MockResourceManagerFacade> secondaries =
+ interceptor.getSecondaryRMs();
+
+ // Increase the attemptId and create a new intercepter instance for it
+ attemptId = ApplicationAttemptId.newInstance(
+ attemptId.getApplicationId(), attemptId.getAttemptId() + 1);
+
+ interceptor = new TestableFederationInterceptor(null, secondaries);
+ interceptor.init(new AMRMProxyApplicationContextImpl(nmContext,
+ getConf(), attemptId, "test-user", null, null, null, registry));
+ registerResponse = interceptor.registerApplicationMaster(registerReq);
+
+ // Should re-attach secondaries and get the three running containers
+ Assert.assertEquals(1, interceptor.getUnmanagedAMPoolSize());
+ Assert.assertEquals(numberOfContainers,
+ registerResponse.getContainersFromPreviousAttempts().size());
+
+ // Release all containers
+ releaseContainersAndAssert(
+ registerResponse.getContainersFromPreviousAttempts());
+
+ // Finish the application
+ FinishApplicationMasterRequest finishReq =
+ Records.newRecord(FinishApplicationMasterRequest.class);
+ finishReq.setDiagnostics("");
+ finishReq.setTrackingUrl("");
+ finishReq.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
+
+ FinishApplicationMasterResponse finshResponse =
+ interceptor.finishApplicationMaster(finishReq);
+ Assert.assertNotNull(finshResponse);
+ Assert.assertEquals(true, finshResponse.getIsUnregistered());
+ return null;
+ }
+ });
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
index d4b8735..23c80ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestableFederationInterceptor.java
@@ -44,6 +44,15 @@ public class TestableFederationInterceptor extends FederationInterceptor {
private AtomicInteger runningIndex = new AtomicInteger(0);
private MockResourceManagerFacade mockRm;
+ public TestableFederationInterceptor() {
+ }
+
+ public TestableFederationInterceptor(MockResourceManagerFacade homeRM,
+ ConcurrentHashMap<String, MockResourceManagerFacade> secondaries) {
+ mockRm = homeRM;
+ secondaryResourceManagers = secondaries;
+ }
+
@Override
protected UnmanagedAMPoolManager createUnmanagedAMPoolManager(
ExecutorService threadPool) {
@@ -68,7 +77,7 @@ public class TestableFederationInterceptor extends FederationInterceptor {
// We create one instance of the mock resource manager per sub cluster. Keep
// track of the instances of the RMs in the map keyed by the sub cluster id
synchronized (this.secondaryResourceManagers) {
- if (this.secondaryResourceManagers.contains(subClusterId)) {
+ if (this.secondaryResourceManagers.containsKey(subClusterId)) {
return (T) this.secondaryResourceManagers.get(subClusterId);
} else {
// The running index here is used to simulate different RM_EPOCH to
@@ -91,6 +100,15 @@ public class TestableFederationInterceptor extends FederationInterceptor {
}
}
+ protected MockResourceManagerFacade getHomeRM() {
+ return mockRm;
+ }
+
+ protected ConcurrentHashMap<String, MockResourceManagerFacade>
+ getSecondaryRMs() {
+ return secondaryResourceManagers;
+ }
+
/**
* Extends the UnmanagedAMPoolManager and overrides methods to provide a
* testable implementation of UnmanagedAMPoolManager.
@@ -104,9 +122,9 @@ public class TestableFederationInterceptor extends FederationInterceptor {
@Override
public UnmanagedApplicationManager createUAM(Configuration conf,
ApplicationId appId, String queueName, String submitter,
- String appNameSuffix) {
+ String appNameSuffix, boolean keepContainersAcrossApplicationAttempts) {
return new TestableUnmanagedApplicationManager(conf, appId, queueName,
- submitter, appNameSuffix);
+ submitter, appNameSuffix, keepContainersAcrossApplicationAttempts);
}
}
@@ -119,8 +137,9 @@ public class TestableFederationInterceptor extends FederationInterceptor {
public TestableUnmanagedApplicationManager(Configuration conf,
ApplicationId appId, String queueName, String submitter,
- String appNameSuffix) {
- super(conf, appId, queueName, submitter, appNameSuffix);
+ String appNameSuffix, boolean keepContainersAcrossApplicationAttempts) {
+ super(conf, appId, queueName, submitter, appNameSuffix,
+ keepContainersAcrossApplicationAttempts);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 7a225c8..8245fd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.service.AbstractService;
@@ -959,9 +960,10 @@ public class MiniYARNCluster extends CompositeService {
protected void initializePipeline(ApplicationAttemptId applicationAttemptId,
String user, Token<AMRMTokenIdentifier> amrmToken,
Token<AMRMTokenIdentifier> localToken,
- Map<String, byte[]> recoveredDataMap, boolean isRecovery) {
+ Map<String, byte[]> recoveredDataMap, boolean isRecovery,
+ Credentials credentials) {
super.initializePipeline(applicationAttemptId, user, amrmToken,
- localToken, recoveredDataMap, isRecovery);
+ localToken, recoveredDataMap, isRecovery, credentials);
RequestInterceptor rt = getPipelines()
.get(applicationAttemptId.getApplicationId()).getRootInterceptor();
// The DefaultRequestInterceptor will generally be the last
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index ef0f713..40d46cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -141,6 +141,8 @@ The figure shows a sequence diagram for the following job execution flow:
b. The security tokens are also modified by the NM when launching the AM, so that the AM can only talk with the AMRMProxy. Any future communication from AM to the YARN RM is mediated by the AMRMProxy.
7. The AM will then request containers using the locality information exposed by HDFS.
8. Based on a policy the AMRMProxy can impersonate the AM on other sub-clusters, by submitting an Unmanaged AM, and by forwarding the AM heartbeats to relevant sub-clusters.
+ a. Federation supports multiple application attempts with AMRMProxy HA. AM containers will have different attempt id in home sub-cluster, but the same Unmanaged AM in secondaries will be used across attempts.
+ b. When AMRMProxy HA is enabled, UAM token will be stored in Yarn Registry. In the registerApplicationMaster call of each application attempt, AMRMProxy will go fetch existing UAM tokens from registry (if any) and re-attached to the existing UAMs.
9. The AMRMProxy will use both locality information and a pluggable policy configured in the state-store to decide whether to forward the resource requests received by the AM to the Home RM or to one (or more) Secondary RMs. In Figure 1, we show the case in which the AMRMProxy decides to forward the request to the secondary RM.
10. The secondary RM will provide the AMRMProxy with valid container tokens to start a new container on some node in its sub-cluster. This mechanism ensures that each sub-cluster uses its own security tokens and avoids the need for a cluster wide shared secret to create tokens.
11. The AMRMProxy forwards the allocation response back to the AM.
@@ -262,16 +264,17 @@ These are extra configurations that should appear in the **conf/yarn-site.xml**
| Property | Example | Description |
|:---- |:---- |
-| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled.
-|`yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor.
+| `yarn.nodemanager.amrmproxy.enabled` | `true` | Whether or not the AMRMProxy is enabled. |
+| `yarn.nodemanager.amrmproxy.interceptor-class.pipeline` | `org.apache.hadoop.yarn.server.nodemanager.amrmproxy.FederationInterceptor` | A comma-separated list of interceptors to be run at the amrmproxy. For federation the last step in the pipeline should be the FederationInterceptor. |
| `yarn.client.failover-proxy-provider` | `org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider` | The class used to connect to the RMs by looking up the membership information in federation state-store. This must be set if federation is enabled, even if RM HA is not enabled.|
Optional:
| Property | Example | Description |
|:---- |:---- |
-|`yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. |
-|`yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. |
+| `yarn.nodemanager.amrmproxy.ha.enable` | `true` | Whether or not the AMRMProxy HA is enabled for multiple application attempt suppport. |
+| `yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. |
+| `yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. |
Running a Sample Job
--------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/49] hadoop git commit: HADOOP-15046. Document Apache Hadoop does
not support Java 9 in BUILDING.txt. Contributed by Hanisha Koneru.
Posted by su...@apache.org.
HADOOP-15046. Document Apache Hadoop does not support Java 9 in BUILDING.txt. Contributed by Hanisha Koneru.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ed44f25
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ed44f25
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ed44f25
Branch: refs/heads/YARN-5881
Commit: 0ed44f25653ad2d97e2726140a7f77a555c40471
Parents: 659e85e
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Nov 22 01:07:42 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Nov 22 01:07:42 2017 +0900
----------------------------------------------------------------------
BUILDING.txt | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ed44f25/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index 9955563..dec3011 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -4,7 +4,7 @@ Build instructions for Hadoop
Requirements:
* Unix System
-* JDK 1.8+
+* JDK 1.8
* Maven 3.3 or later
* ProtocolBuffer 2.5.0
* CMake 3.1 or newer (if compiling native code)
@@ -344,7 +344,7 @@ Building on Windows
Requirements:
* Windows System
-* JDK 1.8+
+* JDK 1.8
* Maven 3.0 or later
* ProtocolBuffer 2.5.0
* CMake 3.1 or newer
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md
new file mode 100644
index 0000000..b974ea8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committer_architecture.md
@@ -0,0 +1,1951 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# S3A Committers: Architecture and Implementation
+
+<!-- DISABLEDMACRO{toc|fromDepth=0|toDepth=5} -->
+
+This document covers the architecture and implementation details of the S3A committers.
+
+For information on using the committers, see [the S3A Committers](./committer.html).
+
+
+## Problem: Efficient, reliable commits of work to consistent S3 buckets
+
+
+The standard commit algorithms (the `FileOutputCommitter` and its v1 and v2 algorithms)
+rely on directory rename being an `O(1)` atomic operation: callers output their
+work to temporary directories in the destination filesystem, then
+rename these directories to the final destination as way of committing work.
+This is the perfect solution for commiting work against any filesystem with
+consistent listing operations and where the `FileSystem.rename()` command
+is an atomic `O(1)` operation.
+
+Using rename allows individual tasks to work in temporary directories, with the
+rename as the atomic operation can be used to explicitly commit tasks and
+ultimately the entire job. Because the cost of the rename is low, it can be
+performed during task and job commits with minimal delays. Note that HDFS
+will lock the namenode metadata during the rename operation, so all rename() calls
+will be serialized. However, as they only update the metadata of two directory
+entries, the duration of the lock is low.
+
+In contrast to a "real" filesystem, Amazon's S3A object store, similar to
+most others, does not support `rename()` at all. A hash operation on the filename
+determines the location of of the data —there is no separate metadata to change.
+To mimic renaming, the Hadoop S3A client has to copy the data to a new object
+with the destination filename, then delete the original entry. This copy
+can be executed server-side, but as it does not complete until the in-cluster
+copy has completed, it takes time proportional to the amount of data.
+
+The rename overhead is the most visible issue, but it is not the most dangerous.
+That is the fact that path listings have no consistency guarantees, and may
+lag the addition or deletion of files.
+If files are not listed, the commit operation will *not* copy them, and
+so they will not appear in the final output.
+
+The solution to this problem is closely coupled to the S3 protocol itself:
+delayed completion of multi-part PUT operations
+
+That is: tasks write all data as multipart uploads, *but delay the final
+commit action until until the final, single job commit action.* Only that
+data committed in the job commit action will be made visible; work from speculative
+and failed tasks will not be instiantiated. As there is no rename, there is no
+delay while data is copied from a temporary directory to the final directory.
+The duration of the commit will be the time needed to determine which commit operations
+to construct, and to execute them.
+
+
+## Terminology
+
+* *Job*: a potentially parallelized query/operation to execute. The execution
+of a job: the division of work into tasks and the management of their completion,
+is generally executed in a single process.
+
+The output of a Job is made visible to other stages in a larger operation
+sequence or other applications if the job *completes successfully*.
+
+* *Job Driver*. Not sure quite what term to use here. Whatever process schedules
+task execution, tracks success/failures and, determines when all the work has been
+processed and then commits the output. It may also determine that a job
+has failed and cannot be recovered, in which case the job is aborted.
+In MR and Tez, this is inside the YARN application master.
+In Spark it is the driver, which can run in the AM, the YARN client, or other
+places (e.g Livy?).
+
+* *Final directory*: the directory into which the output of a job is placed
+so as to be visible.
+
+* *Task* a single operation within a job, on a single process, one which generates
+one or more files.
+After a successful job completion, the data MUST be visible in the final directory.
+A task completes successfully if it generates all the output it expects to without
+failing in some way (error in processing; network/process failure).
+
+* *Job Context* an instance of the class `org.apache.hadoop.mapreduce.JobContext`,
+which provides a read-only view of the Job for the Job Driver and tasks.
+
+* *Task Attempt Context* an instance of the class
+`org.apache.hadoop.mapreduce.TaskAttemptContext extends JobContext, Progressable`,
+which provides operations for tasks, such as getting and setting status,
+progress and counter values.
+
+* *Task Working Directory*: a directory for exclusive access by a single task,
+into which uncommitted work may be placed.
+
+* *Task Commit* The act of taking the output of a task, as found in the
+Task Working Directory, and making it visible in the final directory.
+This is traditionally implemented via a `FileSystem.rename()` call.
+
+ It is useful to differentiate between a *task-side commit*: an operation performed
+ in the task process after its work, and a *driver-side task commit*, in which
+ the Job driver perfoms the commit operation. Any task-side commit work will
+ be performed across the cluster, and may take place off the critical part for
+ job execution. However, unless the commit protocol requires all tasks to await
+ a signal from the job driver, task-side commits cannot instantiate their output
+ in the final directory. They may be used to promote the output of a successful
+ task into a state ready for the job commit, addressing speculative execution
+ and failures.
+
+* *Job Commit* The act of taking all successfully completed tasks of a job,
+and committing them. This process is generally non-atomic; as it is often
+a serialized operation at the end of a job, its performance can be a bottleneck.
+
+* *Task Abort* To cancel a task such that its data is not committed.
+
+* *Job Abort* To cancel all work in a job: no task's work is committed.
+
+* *Speculative Task Execution/ "Speculation"* Running multiple tasks against the same
+input dataset in parallel, with the first task which completes being the one
+which is considered successful. Its output SHALL be committed; the other task
+SHALL be aborted. There's a requirement that a task can be executed in parallel,
+and that the output of a task MUST NOT BE visible until the job is committed,
+at the behest of the Job driver. There is the expectation that the output
+SHOULD BE the same on each task, though that MAY NOT be the case. What matters
+is if any instance of a speculative task is committed, the output MUST BE
+considered valid.
+
+There is an expectation that the Job Driver and tasks can communicate: if a task
+performs any operations itself during the task commit phase, it shall only do
+this when instructed by the Job Driver. Similarly, if a task is unable to
+communicate its final status to the Job Driver, it MUST NOT commit is work.
+This is very important when working with S3, as some network partitions could
+isolate a task from the Job Driver, while the task retains access to S3.
+
+## The execution workflow
+
+
+**setup**:
+
+* A job is created, assigned a Job ID (YARN?).
+* For each attempt, an attempt ID is created, to build the job attempt ID.
+* `Driver`: a `JobContext` is created/configured
+* A committer instance is instantiated with the `JobContext`; `setupJob()` invoked.
+
+
+## The `FileOutputCommitter`
+
+The standard commit protocols are implemented in
+ `org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter`.
+
+There are two algorithms, the "v1" designed to address failures and restarts
+of the MapReduce application master. The V2 algorithm cannot recover from failure
+except by re-executing the entire job. It does, however, propagate all its work
+to the output directory in the task commit. When working with object stores which
+mimic `rename()` by copy and delete, it is more efficient due to the reduced
+number of listings, copies and deletes, and, because these are executed in
+task commits, eliminates the final `O(data)` pause at the end of all work.
+It is still highly inefficient, but the inefficiencies are less visible as
+large pauses in execution.
+
+
+Notes
+
+* The v1 algorithm was implemented to handle MapReduce AM restarts, it was
+ not used in Hadoop 1.x, whose JobTracker could not recover from failures.
+ Historically then, it is the second version of a file commit algorithm.
+* Because the renames are considered to be fast, there is no logging
+of renames being in progress, or their duration.
+* Speculative execution is supported by having every task attempt write its
+uncommitted data to a task attempt directory. When a task is ready to commit,
+it must notify that job driver that it is ready to commit -the job driver
+will then commit or abort the task.
+
+## Hadoop MR Commit algorithm "1"
+
+
+The "v1" commit algorithm is the default commit algorithm in Hadoop 2.x;
+it was implemented as part of [MAPREDUCE-2702](https://issues.apache.org/jira/browse/MAPREDUCE-2702).
+
+This algorithm is designed to handle a failure and restart of the Job driver,
+with the restarted job driver only rerunning the incomplete tasks; the
+output of the completed tasks is recovered for commitment when the restarted
+job completes.
+
+There is a cost: the time to commit by recursive listing all files in all
+committed task directories, and renaming this.
+
+As this is performed sequentially, time to commit is `O(files)`, which is
+generally `O(tasks)`.
+
+
+
+```python
+#Job Attempt Path is `$dest/_temporary/$appAttemptId/`
+
+jobAttemptPath = '$dest/_temporary/$appAttemptId/'
+
+# Task Attempt Path is `$dest/_temporary/$appAttemptId/_temporary/$taskAttemptID`
+taskAttemptPath = '$dest/_temporary/$appAttemptId/_temporary/$taskAttemptID'
+
+#Task committed path is `$dest/_temporary/$appAttemptId/$taskAttemptID`
+taskCommittedPath = '$dest/_temporary/$appAttemptId/$taskAttemptID'
+```
+
+Tasks write in/under the task attempt path.
+
+### Job Setup
+
+```python
+fs.mkdir(jobAttemptPath)
+```
+
+### Task Setup
+
+None: directories are created on demand.
+
+
+### Task Commit
+
+Rename task attempt path to task committed path.
+
+```python
+
+def needsTaskCommit(fs, jobAttemptPath, taskAttemptPath, dest):
+ return fs.exists(taskAttemptPath)
+
+
+def commitTask(fs, jobAttemptPath, taskAttemptPath, dest):
+ if fs.exists(taskAttemptPath) :
+ fs.delete(taskCommittedPath, recursive=True)
+ fs.rename(taskAttemptPath, taskCommittedPath)
+```
+
+On a genuine fileystem this is an `O(1)` directory rename.
+
+On an object store with a mimiced rename, it is `O(data)` for the copy,
+along with overhead for listing and deleting all files (For S3, that's
+`(1 + files/500)` lists, and the same number of delete calls.
+
+
+### Task Abort
+
+Delete task attempt path.
+
+```python
+def abortTask(fs, jobAttemptPath, taskAttemptPath, dest):
+ fs.delete(taskAttemptPath, recursive=True)
+```
+
+On a genuine fileystem this is an `O(1)` operation. On an object store,
+proportional to the time to list and delete files, usually in batches.
+
+
+### Job Commit
+
+Merge all files/directories in all task commited paths into final destination path.
+Optionally; create 0-byte `_SUCCESS` file in destination path.
+
+```python
+def commitJob(fs, jobAttemptDir, dest):
+ for committedTask in fs.listFiles(jobAttemptDir):
+ mergePathsV1(fs, committedTask, dest)
+ fs.touch("$dest/_SUCCESS")
+```
+
+(See below for details on `mergePaths()`)
+
+
+A failure during job abort cannot be recovered from except by re-executing
+the entire query:
+
+```python
+def isCommitJobRepeatable() :
+ return True
+```
+
+Accordingly, it is a failure point in the protocol. With a low number of files
+and fast rename/list algorithms, the window of vulnerability is low. At
+scale, the vulnerability increases. It could actually be reduced through
+parallel execution of the renaming of of committed tasks.
+
+
+### Job Abort
+
+Delete all data under job attempt path.
+
+```python
+def abortJob(fs, jobAttemptDir, dest):
+ fs.delete(jobAttemptDir, recursive = True)
+```
+
+### Job Cleanup
+
+```python
+def cleanupJob(fs, dest):
+ fs.delete('$dest/_temporary', recursive = True)
+```
+
+
+### Job Recovery
+
+1. Data under task committed paths is retained
+1. All directories under `$dest/_temporary/$appAttemptId/_temporary/` are deleted.
+
+Uncommitted/unexecuted tasks are (re)executed.
+
+This significantly improves time to recover from Job driver (here MR AM) failure.
+The only lost work is that of all tasks in progress -those which had generated
+data but were not yet committed.
+
+Only the failure of the job driver requires a job restart, not an individual
+task. Therefore the probability of this happening is independent of the number
+of tasks executed in parallel, instead simply due to the duration of the query.
+
+The longer the task, the higher the risk of failure, the more value there is
+in recovering the work in progress.
+
+Fast queries not only have a lower risk of failure, they can recover from
+failure simply by rerunning the entire job. This is implicitly the strategy
+in Spark, which does not attempt to recover any in-progress jobs. The faster
+your queries, the simpler your recovery strategy needs to be.
+
+### `mergePaths(FileSystem fs, FileStatus src, Path dest)` Algorithm
+
+`mergePaths()` is the core algorithm to merge data; it is somewhat confusing
+as the implementation mixes the strategies for both algorithms across
+two co-recursive routines, `mergePaths()` and `renameOrMerge()`.
+
+
+Here the two algorithms have been split, and one of the co-recursive methods
+inlined.
+
+```python
+def mergePathsV1(fs, src, dest) :
+ if fs.exists(dest) :
+ toStat = fs.getFileStatus(dest)
+ else:
+ toStat = None
+
+ if src.isFile :
+ if not toStat is None :
+ fs.delete(dest, recursive = True)
+ fs.rename(src.getPath, dest)
+ else :
+ # destination is directory, choose action on source type
+ if src.isDirectory :
+ if not toStat is None :
+ if not toStat.isDirectory :
+ # Destination exists and is not a directory
+ fs.delete(dest)
+ fs.rename(src.getPath(), dest)
+ else :
+ # Destination exists and is a directory
+ # merge all children under destination directory
+ for child in fs.listStatus(src.getPath) :
+ mergePathsV1(fs, child, dest + child.getName)
+ else :
+ # destination does not exist
+ fs.rename(src.getPath(), dest)
+```
+
+## v2 commit algorithm
+
+
+The v2 algorithm directly commits task output into the destination directory.
+It is essentially a re-implementation of the Hadoop 1.x commit algorithm.
+
+1. During execution, intermediate data becomes visible.
+1. On a failure, all output must be deleted and the job restarted.
+
+It implements `mergePaths` differently, as shown below.
+
+```python
+def mergePathsV2(fs, src, dest) :
+ if fs.exists(dest) :
+ toStat = fs.getFileStatus(dest)
+ else:
+ toStat = None
+
+ if src.isFile :
+ if not toStat is None :
+ fs.delete(dest, recursive = True)
+ fs.rename(src.getPath, dest)
+ else :
+ # destination is directory, choose action on source type
+ if src.isDirectory :
+ if not toStat is None :
+ if not toStat.isDirectory :
+ # Destination exists and is not a directory
+ fs.delete(dest)
+ fs.mkdirs(dest) #
+ for child in fs.listStatus(src.getPath) : # HERE
+ mergePathsV2(fs, child, dest + child.getName) #
+
+ else :
+ # Destination exists and is a directory
+ # merge all children under destination directory
+ for child in fs.listStatus(src.getPath) :
+ mergePathsV2(fs, child, dest + child.getName)
+ else :
+ # destination does not exist
+ fs.mkdirs(dest) #
+ for child in fs.listStatus(src.getPath) : # HERE
+ mergePathsV2(fs, child, dest + child.getName) #
+```
+
+Both recurse down any source directory tree, and commit single files
+by renaming the files.
+
+A a key difference is that the v1 algorithm commits a source directory to
+via a directory rename, which is traditionally an `O(1)` operation.
+
+In constrast, the v2 algorithm lists all direct children of a source directory
+and recursively calls `mergePath()` on them, ultimately renaming the individual
+files. As such, the number of renames it performa equals the number of source
+*files*, rather than the number of source *directories*; the number of directory
+listings being `O(depth(src))` , where `depth(path)` is a function returning the
+depth of directories under the given path.
+
+On a normal filesystem, the v2 merge algorithm is potentially more expensive
+than the v1 algorithm. However, as the merging only takes place in task commit,
+it is potentially less of a bottleneck in the entire execution process.
+
+On an objcct store, it is suboptimal not just from its expectation that `rename()`
+is an `O(1)` operation, but from its expectation that a recursive tree walk is
+an efficient way to enumerate and act on a tree of data. If the algorithm was
+switched to using `FileSystem.listFiles(path, recursive)` for a single call to
+enumerate all children under a path, then the listing operation would be significantly
+faster, at least on a deep or wide tree. However, for any realistic dataset,
+the size of the output files is likely to be the main cause of delays. That
+is, if the cost of `mergePathsV2` is `O(depth(src)) + O(data))`, then
+generally the `O(data)` value will be more significant than the `depth(src)`.
+
+There is one key exception: tests which work on small amounts of data yet try
+to generate realistic output directory structures. In these tests the cost
+of listing directories and calling `getFileStatus()` could exceed that of the copy
+calls. This is why small-scale tests of the commit algorithms against object stores
+must be considered significantly misleading.
+
+### v2 Task Commit
+
+Rename task attempt path to task committed path.
+
+```python
+def needsTaskCommit(fs, jobAttemptPath, taskAttemptPath, dest):
+ return fs.exists(taskAttemptPath)
+
+def commitTask(fs, jobAttemptPath, taskAttemptPath, dest):
+ if fs.exists(taskAttemptPath) :
+ mergePathsV2(fs. taskAttemptPath, dest)
+```
+
+### v2 Task Abort
+
+Delete task attempt path.
+
+```python
+def abortTask(fs, jobAttemptPath, taskAttemptPath, dest):
+ fs.delete(taskAttemptPath, recursive=True)
+```
+
+Cost: `O(1)` for normal filesystems, `O(files)` for object stores.
+
+
+### v2 Job Commit
+
+As all the task output is already completed, all that is needed is
+to touch the `_SUCCESS` marker.
+
+```python
+def commitJob(fs, jobAttemptDir, dest):
+ fs.touch("$dest/_SUCCESS")
+```
+Cost: `O(1)`
+
+A failure during job abort is implicitly repeatable
+
+```python
+def isCommitJobRepeatable() :
+ return True
+```
+
+### v2 Job Abort
+
+Delete all data under job attempt path.
+
+```python
+def abortJob(fs, jobAttemptDir, dest):
+ fs.delete(jobAttemptDir, recursive=True)
+```
+
+Cost: `O(1)` for normal filesystems, `O(files)` for object stores.
+
+### v2 Task Recovery
+
+As no data is written to the destination directory, a task can be cleaned up
+by deleting the task attempt directory.
+
+### v2 Job Recovery
+
+Because the data has been renamed into the destination directory, it is nominally
+recoverable. However, this assumes that the number and name of generated
+files are constant on retried tasks.
+
+
+## How MapReduce uses the committer in a task
+
+MapReduce runs each Mapper or Reducer in its own container; it
+gets its own process. Implicitly, this also means that each task gets
+its own instance of every filesystem.
+
+The work is choreographed in `org.apache.hadoop.mapred.Task`, for which there
+are specific subclassing for the Map (`MapTask`) and reduce (`ReduceTask`).
+There are also cleanup tasks whose role is simpler: clean things up,
+and a Job setup task which runs `OutputCommittre.setupJob`. That is: even
+the job setup phase is run in a Task.
+
+MapTask uses the committer to write the output of all the mappers into
+the filesystem, ready for the reducers. Each partition writes its data
+as a `MapFile`, which is actually two `SequenceFile` files in a directory,:
+the `data` file of all Key-Value
+output, and `index` which contains an index of some of the keys in the file.
+
+This is all written to the local filesystem.
+
+The `ReduceTask` does the final write to the destination filesystem.
+
+Because the Map phase uses a committer to commits intermediate work,
+any plug in committer supplied to a process
+through any extension mechanism *must* work with the output generated by a mapper.
+The staging committers do only if unique filenames are disabled, but as they
+and the magic committers are only meant to be used for the final output of work,
+it is somewhat moot. What is important is to be able to use different committers
+for the map phase as the final reduce. This is implicit if separate committers
+are defined for different filesystems: the new committer can be defined for
+the final destination FS, while `file://` can retain the default
+`FileOutputCommitter`.
+
+### Task Setup
+
+`Task.initialize()`: read in the configuration, instantate the `JobContextImpl`
+and `TaskAttemptContextImpl` instances bonded to the current job & task.
+
+### Task Ccommit
+
+After the work is completed, `Task.done()` is invoked. Which is essentially the following
+codepath:
+
+```java
+if (committer.needsTaskCommit(taskContext)) {
+ // get permission to commit from AM
+ int retries = MAX_RETRIES;
+ while(true) {
+ try {
+ umbilical.commitPending(taskId, taskStatus);
+ break;
+ } catch(IOException ie) {
+ if (--retries == 0) {
+ // FAIL WITHOUT CALLING ABORT
+ System.exit(67);
+ }
+ }
+ }
+ // commit the work
+ try {
+ committer.commitTask(taskContext);
+ } catch (IOException iee) {
+ // failure: abort
+ try {
+ committer.abortTask(taskContext);
+ } catch (IOException ioe) {
+ LOG.warn("Failure cleaning up: " +
+ StringUtils.stringifyException(ioe));
+ }
+ throw iee;
+ }
+}
+```
+
+That is: if and only if there is data to write, the Task requests clearance
+to do so from the AM. This ensures that speculative work is not committed if
+another task has written it, *and that even non-speculative work is not committed
+if the task has lost contact with the AM*.
+
+That is: task output is not committed if the Job's AM has failed or
+is in a separate network partition from the Task.
+
+If permission is granted, the commit call is invoked.
+If this doesn't work, `abortTask()` is called; all
+failures there are logged and swallowed.
+
+
+This method actually appears to be limited in that it does not guarantee
+that `committer.abortTask()` is always called: if the `umbilical.commitPending()` calls
+fail repeatedly and the Task process aborts, `committer.abortTask()` is not called.
+If this is due to a network partition, we can hope that the AM invokes
+`committer.abortTask()`, and that if it is an AM failure then a restarted AM can
+clean up its previous attempts. For the classic FileOutputCommitter, listing and
+deleting the previous attempt's data is straightforward. However, for S3 committers
+using Multipart Upload as the means of uploading uncommitted data, it is critical
+to ensure that pending uploads are always aborted. This can be done by
+
+* Making sure that all task-side failure branvches in `Task.done()` call `committer.abortTask()`.
+* Having job commit & abort cleaning up all pending multipart writes to the same directory
+tree. That is: require that no other jobs are writing to the same tree, and so
+list all pending operations and cancel them.
+* Add a CLI command to verify that there are no commits under a path; returning
+a non-zero exit code if there are. This can be used in testing.
+* Add a CLI command to purge commits under a path. This can be used in workflows,
+if felt necessary, and in test runs to isolate the output of different tests.
+* Document a recommended purge-pending-commit timeout for the system
+* Add probes in the integration tests to verify that pending uploads are never
+present after a write.
+
+
+
+### How MapReduce uses the committer in the Application Master
+
+The AM uses a committer to set up and commit the job. To support failure
+and recovery of the AM, `OutputCommitter.isRecoverySupported()` is used
+to declare whether all the output of successful tasks can be used in the final
+job, or the entire job needs to be reset and repeated.
+`OutputCommitter.isCommitJobRepeatable()` addresses the other question:
+can a committer recover from a failure of the commit process itself.
+
+A committer is created in the Application Master, and handed to an instance
+of `org.apache.hadoop.mapreduce.v2.app.commit.CommitterEventHandler`,
+which then manages the job-level lifecycle.
+
+If the MRv1 API is used, the committer is chosen from the value of
+`"mapred.output.committer.class"`; in the MRv2 API the output format
+is instantiated, then asked for a committer using a task and task attempt ID of
+0. A committer is obtained from the output format via a call to
+`Committer.getOutputCommitter(taskContext)`, again using the task attempt context
+with the (job, 0, 0) task and attempt IDs. That is: even for the job committers,
+a task context is always passed in to the `OutputFormat` when requesting a committer.
+*It is critical for all implementations of `OutputCommitter.abortTask()` to
+be able to execute from the AM, rather than the container and host running
+the task. Furthermore, all information needed for the abort (paths, filesystem instances
+&c) *must* be retrieved from the `TaskAttemptContext` passed to the method,
+rather than relying on fields initiated from the context passed to the constructor.
+
+
+#### AM: Job setup: `OutputCommitter.setupJob()`
+
+This is initated in `org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl.StartTransition`.
+It is queued for asynchronous execution in `org.apache.hadoop.mapreduce.v2.app.MRAppMaster.startJobs()`,
+which is invoked when the service is started. Thus: the job is set up when the
+AM is started.
+
+
+#### AM: Job Commit: `OutputCommitter.commitJob()`
+
+In the "default"/cluster filesystem, the `CommitterEventHandler` uses data in
+the staging area defined in `yarn.app.mapreduce.am.staging-dir`
+(default `/tmp/hadoop-yarn/staging/$user/.staging`), in a subdirectory
+named from the job ID.
+
+Three paths are built up for using the filesystem for tracking the state
+of the commit process, with a goal of making the commit operation recoverable,
+where supported.
+
+
+| name | role |
+|------|--------|
+| `COMMIT_STARTED` | mark the commencement of the job commit |
+| `COMMIT_SUCCESS` | mark the successful completion of the job commit |
+| `COMMIT_FAIL` | mark the failure of the job commit |
+
+These markers are used to manage job restart/failure of they happen during
+the job commit itself.
+
+When an AM starts up, it looks in its staging area for these files. as a way
+to determine the previous state of the job. If there are no `COMMIT_` marker files,
+the job is considered not to have attempted to commit itself yet.
+
+
+The presence of `COMMIT_SUCCESS` or `COMMIT_FAIL` are taken as evidence
+that the previous job completed successfully or unsucessfully; the AM
+then completes with a success/failure error code, without attempting to rerun
+the job.
+
+If `COMMIT_STARTED` exists but not either of the completion markers, then,
+if the committer declares that its job commit operation is repeatable
+(`Committer.isCommitJobRepeatable(jobContext) == true`), then an attempt
+is made to recommit the job, deleting the `COMMIT_STARTED` and commencing
+the commit process again.
+
+These `COMMIT_STARTED` files are simply 0-byte files, but are created
+with the overwrite bit only set to true if the job commit is considered
+repeatable:
+
+```java
+private void touchz(Path p, boolean overwrite) throws IOException {
+ fs.create(p, overwrite).close();
+}
+```
+That is: the atomicity of the `create(path, overwrite=false)` on the cluster
+filesystem is used to guarantee that only one process will attempt to commit
+a specific job.
+
+```java
+boolean commitJobIsRepeatable = committer.isCommitJobRepeatable(
+ event.getJobContext());
+try {
+ touchz(startCommitFile, commitJobIsRepeatable);
+ waitForValidCommitWindow();
+ committer.commitJob(event.getJobContext());
+ touchz(endCommitSuccessFile, commitJobIsRepeatable);
+} catch (Exception e) {
+ touchz(endCommitFailureFile, commitJobIsRepeatable);
+}
+```
+
+The `waitForValidCommitWindow()` operation is important: it declares that
+the committer must not commit unless there has been communication with the YARN
+Resource Manager with in `yarn.app.mapreduce.am.job.committer.commit-window` milliseconds
+(default: 10,000). It does this by waiting until the next heartbeat it received.
+There's a possible bug here: if the interval is set too small the thread may
+permanently spin waiting a callback within the window. Ignoring that, this algorithm
+guarantees that
+
+1. As only one call can create a file with `overwrite=false`,
+ only one process's attempt to commit a non-repeatable job will proceed
+
+1. Only a process with contact with the YARN within the configured window
+ may commit a job.
+
+1. If the AM is partitioned from the rest of the network, provided that its clock
+is monotonically increasing at the same rate as the rest of the cluster, then
+the rest of the cluster can be confident that
+`yarn.app.mapreduce.am.ob.committer.commit-window` milliseconds after the AM
+successfully heartbeated to the YARN RM, then the output of this job attempt
+will *never* be committed. This permits 1 job to run simultaneously, but
+helps ensure that only one of them will attempt to commit.
+
+1. Provided YARN heartbeats are only sent to the AM which successfully created
+the `COMMIT_STARTED` file, it will initiate the commit operation.
+
+Possible issues with this algorithm:
+
+* The `COMMIT_STARTED` file is created before waiting to get a heartbeat. It
+may be that this AM has lost contact with YARN, but doesn't know it yet. When the
+YARN liveness protocols eventually time out, the AM will correctly terminate,
+but as the `COMMIT_STARTED` file has been created at this point, no other launched
+AM will be able to commit.
+
+* If two committers attempt to create `COMMIT_STARTED` files on a no-repeatable
+commit, one will succeed, wait for a heartbeat then attempt a (possibly slow) commit.
+The second committer will fail, and will *immediately* create a `COMMIT_FAILED` file.
+As a result, the state of the staging area will imply that the commit has failed,
+when really it is in progress, and that only the second process failed.
+
+It would seem safer to address this through
+1. Waiting for a heartbeat before creating the `COMMIT_STARTED` file.
+1. Maybe: not creating the `COMMIT_FAILED` file if the failure happens when
+trying to create the `COMMIT_STARTED` file. That is: only a process which
+successfully created the `COMMIT_STARTED` file may indicate that a commit has failed.
+
+
+### AM: Cancelling job commit
+
+The thread performing the commit is interrupted; the `CommitterEventHandler`
+awaits for it to finish. (set in `yarn.app.mapreduce.am.job.committer.cancel-timeout`
+as milliseconds).
+
+### AM: Task Abort
+
+The AM may call the `OutputCommitter.taskAbort()` with a task attempt context,
+when handling the failure/loss of a container. That is: on container failure,
+the task abort operation is executed in the AM, using the AM's committer.
+This avoids the need to create a new container, and means that the "best-effort"
+task abort does cope with container failures.
+
+A partition between the AM and the task container means that this AM-executed
+task abort may take place while a task in the partitioned container is still
+executing. Unless output writing operations will fail after the abort operation,
+the partitioned task may not become aware of the partition until it's own task
+commit sequence in `Task.done()`, when `talkToAMTGetPermissionToCommit()`
+
+# Requirements of an S3A Committer
+
+1. Support an eventually consistent S3 object store as a reliable direct
+destination of work through the S3A filesystem client.
+1. Efficient: implies no rename, and a minimal amount of delay in the job driver's
+task and job commit phases,
+1. Support task failure and speculation.
+1. Can be used by existing code: Hadoop MapReduce, Spark, Hive.
+1. Retrofittable to existing subclasses of FileOutputFormat and/or compatible
+with committers which expect a specific FileOutputFormat.
+1. Clean up uncommitted data from all task attempts, all previous attempts of
+the job, and any previous incompleted jobs.
+1. Security: not to permit privilege escalation from other users with
+write access to the same file system(s).
+
+
+## Features of S3 and the S3A Client
+
+
+A core problem is that
+[object stores are not filesystems](../../../hadoop-project-dist/hadoop-common/filesystem/introduction.html);
+how `rename()` has been emulated in the S3A client means that both the existing
+MR committer algorithms have significant performance problems.
+
+1. Single-object renames are implemented as a copy and delete sequence.
+1. COPY is atomic, but overwrites cannot be prevented.
+1. Amazon S3 is eventually consistent on listings, deletes and updates.
+1. Amazon S3 has create consistency, however, the negative response of a HEAD/GET
+performed on a path before an object was created can be cached, unintentionally
+creating a create inconsistency. The S3A client library does perform such a check,
+on `create()` and `rename()` to check the state of the destination path, and
+so, whether the operation is permitted.
+1. Multi-object renames are sequential or parallel single object COPY+DELETE operations:
+non atomic, `O(data)` and, on failure, can leave the filesystem in an unknown
+state.
+1. There is a PUT operation, capable of uploading 5GB of data in one HTTP request.
+1. The PUT operation is atomic, but there is no PUT-no-overwrite option.
+1. There is a multipart POST/PUT sequence for uploading larger amounts of data
+in a sequence of PUT requests.
+
+
+The Hadoop S3A Filesystem client supports PUT and multipart PUT for uploading
+data, with the `S3ABlockOutputStream` of HADOOP-13560 uploading written data
+as parts of a multipart PUT once the threshold set in the configuration
+parameter `fs.s3a.multipart.size` (default: 100MB).
+
+[S3Guard](./s3guard.html) adds an option of consistent view of the filesystem
+to all processes using the shared DynamoDB table as the authoritative store of
+metadata. Some S3-compatible object stores are fully consistent; the
+proposed algorithm is designed to work with such object stores without the
+need for any DynamoDB tables.
+
+## Related work: Spark's `DirectOutputCommitter`
+
+One implementation to look at is the
+[`DirectOutputCommitter` of Spark 1.6](https://github.com/apache/spark/blob/branch-1.6/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/parquet/DirectParquetOutputCommitter.scala).
+
+This implements a zero rename commit by subclassing the `ParquetOutputCommitter`
+and then
+
+1. Returning the destination directory as the task working directory.
+1. Subclassing all the task commit/abort operations to be no-ops.
+
+With the working directory as the destination directory, there is no need
+to move/rename the task output on a successful commit. However, it is flawed.
+There is no notion of "committing" or "aborting" a task, hence no ability to
+handle speculative execution or failures. This is why the committer
+was removed from Spark 2 [SPARK-10063](https://issues.apache.org/jira/browse/SPARK-10063)
+
+There is also the issue that work-in-progress data is visible; this may or may
+not be a problem.
+
+## Related work: IBM's "Stocator" committer
+
+IBM's [Stocator](https://github.com/SparkTC/stocator) can transform indirect
+writes of V1/V2 committers into direct writes to the destination directory.
+
+Hpw does it do this? It's a special Hadoop `FileSystem` implementation which
+recognizes writes to `_temporary` paths and translate them to writes to the
+base directory. As well as translating the write operation, it also supports
+a `getFileStatus()` call on the original path, returning details on the file
+at the final destination. This allows for committing applications to verify
+the creation/existence/size of the written files (in contrast to the magic
+committer covdered below).
+
+The FS targets Openstack Swift, though other object stores are supportable through
+different backends.
+
+This solution is innovative in that it appears to deliver the same semantics
+(and hence failure modes) as the Spark Direct OutputCommitter, but which
+does not need any changs in either Spark *or* the Hadoop committers. In contrast,
+the committers proposed here combines changing the Hadoop MR committers for
+ease of pluggability, and offers a new committer exclusivley for S3, one
+strongly dependent upon and tightly integrated with the S3A Filesystem.
+
+The simplicity of the Stocator committer is something to appreciate.
+
+## Background: The S3 multi-part PUT mechanism
+
+In the [S3 REST API](http://docs.aws.amazon.com/AmazonS3/latest/dev/uploadobjusingmpu.html),
+multipart uploads allow clients to upload a series of "Parts" of a file,
+then commit the upload with a final call.
+
+1. Caller initiates a multipart request, including the destination bucket, key
+and metadata.
+
+ POST bucket.s3.aws.com/path?uploads
+
+ An UploadId is returned
+
+1. Caller uploads one or more parts.
+
+ PUT bucket.s3.aws.com/path?partNumber=PartNumber&uploadId=UploadId
+
+ The part number is used to declare the ordering of the PUTs; they
+ can be uploaded in parallel and out of order.
+ All parts *excluding the final part* must be 5MB or larger.
+ Every upload completes with an etag returned
+
+1. Caller completes the operation
+
+ POST /ObjectName?uploadId=UploadId
+ <CompleteMultipartUpload>
+ <Part><PartNumber>(number)<PartNumber><ETag>(Tag)</ETag></Part>
+ ...
+ </CompleteMultipartUpload>
+
+ This final call lists the etags of all uploaded parts and the actual ordering
+ of the parts within the object.
+
+The completion operation is apparently `O(1)`; presumably the PUT requests
+have already uploaded the data to the server(s) which will eventually be
+serving up the data for the final path. All that is needed to complete
+the upload is to construct an object by linking together the files in
+the server's local filesystem and udate an entry the index table of the
+object store.
+
+In the S3A client, all PUT calls in the sequence and the final commit are
+initiated by the same process. *This does not have to be the case*.
+It is that fact, that a different process may perform different parts
+of the upload, which make this algorithm viable.
+
+
+## The Netfix "Staging" committer
+
+Ryan Blue, of Netflix, has submitted an alternate committer, one which has a
+number of appealing features
+
+* Doesn't have any requirements of the destination object store,
+* Known to work.
+
+The final point is not to be underestimated, es not even
+a need for a consistency layer.
+* Overall a simpler design.pecially given the need to
+be resilient to the various failure modes which may arise.
+
+
+The commiter writes task outputs to a temporary directory on the local FS.
+Task outputs are directed to the local FS by `getTaskAttemptPath` and `getWorkPath`.
+On task commit, the committer enumerates files in the task attempt directory (ignoring hidden files).
+Each file is uploaded to S3 using the [multi-part upload API](http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuoverview.html),
+
+The information needed to commit the upload is saved into HDFS and committed
+via that protocol: when the job commits, the pending uploads of the successful
+tasks are all committed.
+
+
+### Commit logic
+
+The core algorithm is as follows:
+
+1. The destination directory for output (e.g. `FileOutputFormat` and subclasses)
+is a local `file://` reference.
+1. Task commit initiates the multipart PUT to the destination object store.
+1. A list of every pending PUT for task is persisted to a single file
+within a consistent, cluster-wide filesystem. For Netflix, that is HDFS.
+1. The Standard `FileOutputCommitter` (algorithm 1) is used to manage the commit/abort of these
+files. That is: it copies only those lists of files to commit from successful tasks
+into a (transient) job commmit directory.
+1. The S3 job committer reads the pending file list for every task committed
+in HDFS, and completes those put requests.
+
+By using `FileOutputCommmitter` to manage the propagation of the lists of files
+to commit, the existing commit algorithm implicitly becomes that defining which
+files will be committed at the end of the job.
+
+
+The Netflix contribution has Hadoop `OutputCommitter` implementations for S3.
+
+There are 3 main classes:
+* `S3MultipartOutputCommitter` is a base committer class that handles commit logic. This should not be used directly.
+* `S3DirectoryOutputCommitter` for writing unpartitioned data to S3 with conflict resolution.
+* `S3PartitionedOutputCommitter` for writing partitioned data to S3 with conflict resolution.
+
+Callers should use `S3DirectoryOutputCommitter` for single-directory outputs,
+or `S3PartitionedOutputCommitter` for partitioned data.
+
+
+These S3 committers work by writing task outputs to a temporary directory on the local FS.
+Task outputs are directed to the local FS by `getTaskAttemptPath` and `getWorkPath`.
+
+
+### Conflict Resolution
+
+The single-directory and partitioned committers handle conflict resolution by
+checking whether target paths exist in S3 before uploading any data.
+There are 3 conflict resolution modes, controlled by setting `fs.s3a.committer.staging.conflict-mode`:
+
+* `fail`: Fail a task if an output directory or partition already exists. (Default)
+* `append`: Upload data files without checking whether directories or partitions already exist.
+* `replace`: If an output directory exists, delete it so the new data replaces the current content.
+
+The partitioned committer enforces the conflict mode when a conflict is detected with output data, not before the job runs.
+Conflict resolution differs from an output mode because it does not enforce the mode when there is no conflict.
+For example, overwriting a partition should remove all sub-partitions and data it contains, whether or not new output is created.
+Conflict resolution will only replace partitions that have output data.
+
+When the conflict mode is `replace`, conflicting directories are removed during
+job commit. Data is only deleted if all tasks have completed successfully.
+
+A UUID that identifies a write is added to filenames that are uploaded to S3.
+This allows rolling back data from jobs that fail during job commit (see failure cases below) and avoids
+file-level conflicts when appending data to existing directories.
+
+
+*Note* the checks for existence are made via `S3AFileSystem.getFileStatus()` requests of the destination paths.
+Unless the view of the S3 store is consistent, it may be that a newly-deleted object
+is still discovered in the probe, so a commit fail, even when there is no longer any actual conflict.
+
+### Performance
+
+Compared to the previous proposal, henceforth the "magic" committer, this
+committer, the "staging committer", adds the extra overhead of uploading
+each file at the end of every task. This is an `O(data)` operation; it can be
+parallelized, but is bounded by the bandwidth from compute node to S3, as
+well as the write/IOP capacity of the destination shard of S3. If many tasks
+complete at or near the same time, there may be a peak of bandwidth load
+slowing down the upload.
+
+Time to commit will be the same, and, given the Netflix committer has already
+implemented the paralellization logic here, a time of `O(files/threads)`.
+
+### Resilience
+
+There's already a lot of code in the task and job commits to handle failure.
+
+Any failure in a commit triggers a best-effort abort/revert of the commit
+actions for a task/job.
+
+Task commits delegate to the `FileOutputCommitter` to ensure that only one task's
+output reaches the job commit.
+
+Similarly, if a task is aborted, temporary output on the local FS is removed.
+
+If a task dies while the committer is running, it is possible for data to be
+eft on the local FS or as unfinished parts in S3.
+Unfinished upload parts in S3 are not visible to table readers and are cleaned
+up following the rules in the target bucket's life-cycle policy.
+
+Failures during job commit are handled by deleting any files that have already
+been completed and aborting the remaining uploads.
+Because uploads are completed individually, the files that are deleted were visible to readers.
+
+If the process dies while the job committer is running, there are two possible failures:
+
+1. Some directories that would be replaced have been deleted, but no new data is visible.
+2. Some new data is visible but is not complete, and all replaced directories have been removed.
+ Only complete files are visible.
+
+If the process dies during job commit, cleaning up is a manual process.
+File names include a UUID for each write so that files can be identified and removed.
+
+
+**Failure during task execution**
+
+All data is written to local temporary files; these need to be cleaned up.
+
+The job must ensure that the local (pending) data is purged.
+
+**Failure during task commit**
+
+
+A process failure during the upload process will result in the
+list of pending multipart PUTs to *not* be persisted to the cluster filesystem.
+This window is smaller than the entire task execution, but still potentially
+significant, at least for large uploads.
+
+Per-file persistence, or incremental overwrites of the upload list may
+reduce the problems here, but there would still be a small risk of
+an outstanding multipart upload not being recorded
+
+**Explicit Task abort before task commit**
+
+Task will delete all local data; no uploads will be initiated.
+
+**Failure to communicate with S3 during data upload**
+
+If an upload fails, tasks will:
+
+* Retry using the retry policies implemented in the S3AFileSystem classes
+and the AWS libraries.
+* Eventually: attempt to abort outstanding multipart uploads already initiated.
+* Remove temporary files on the local FS.
+
+
+**Explicit Job Abort**
+
+All in-progress tasks are aborted and cleaned up. The pending commit data
+of all completed tasks can be loaded, outstanding multipart PUT requests aborted.
+
+
+This is done by
+
+1. Listing all local files, with a best effort read attempt followed by
+an abort of all successfully read files.
+1. List and abort all pending multipart uploads.
+
+Because of action #2, action #1 is superflous. It is retained so as to leave
+open the option of making action #2 a configurable option -which would be
+required to handle the use case of >1 partitioned commit running simultaneously/
+
+**Job Driver failure before Job Commit**
+
+Because the local data is managed with the v1 commit algorithm, the
+second attempt of the job will recover all the outstanding commit data
+of the first attempt; those tasks will not be rerun.
+
+This also ensures that on a job abort, the invidual tasks' .pendingset
+files can be read and used to initiate the abort of those uploads.
+That is: a recovered job can clean up the pending writes of the previous job
+
+If the query engine does not support multiple job attempts, then the
+pending commit data will not be recovered; an explicit abort operation will
+need to be initiated (we will add a CLI command for this), or the S3 bucket
+must be configured to automatically delete the pending request.
+
+**Job Driver failure during Job Commit**
+
+Those uploads already executed by a failed job commit will persist; those
+yet to execute will remain outstanding.
+
+The committer currently declares itself as non-recoverble, but that
+may not actually hold, as the recovery process could be one of:
+
+1. Enumerate all job commits from the .pendingset files (*:= Commits*).
+1. List all outstanding uploads under the destination path (*:= Outstandings*)..
+1. List all written files (for better consistency, via a GET call on the known
+filenames)
+1. Identify all files which are yet to be completed (*Commits - Written*)/
+1. Verify that the set of pending uploads matches (*Outstanding = (Commits - Written)*)
+
+The main problem here is the progress of the job-commit-time conflict resolution
+process: how to determine if it completed, as the only way to be confident
+that all files in the destination directory are to be retained is by knowing
+that the pre-commit phase completed. This could be implicitly determined
+based on the rule "no uploads are committed until precommit is completed".
+If it can be determined that 1+ upload has completed, then it could be inferred
+that precommit had completed and so the job could be repeated.
+
+This is dangerous territory to delve into. For now, the committer declares
+itself as unrecoverable.
+
+**Entire application failure before any task commit**
+
+Data is left on local systems, in the temporary directories. This may
+not be cleaned up.
+
+**Entire application failure after one or more task commits, before job commit**
+
+* A multipart PUT request will be outstanding for every pending write.
+* A temporary directory in HDFS will list all known pending requests.
+
+**Job complete/abort after >1 task failure**
+
+1. All pending put data listed in the job completion directory needs to be loaded
+and then cancelled.
+1. Any other pending writes to the dest dir need to be enumerated and aborted.
+This catches the situation of a task failure before the output is written.
+1. All pending data in local dirs need to be deleted.
+
+Issue: what about the destination directory: overwrite or not? It could well
+depend upon the merge policy.
+
+
+#### Overall Resilience
+
+1. The only time that incomplete work will appear in the destination directory
+is if the job commit operation fails partway through.
+1. There's a risk of leakage of local filesystem data; this will need to
+be managed in the response to a task failure.
+1. There's a risk of uncommitted multipart PUT operations remaining outstanding,
+operations which will run up bills until cancelled. (as indeed, so does the Magic Committer).
+
+
+For cleaning up PUT commits, as well as scheduled GC of uncommitted writes, we
+may want to consider having job setup list and cancel all pending commits
+to the destination directory, on the assumption that these are from a previous
+incomplete operation.
+
+We should add a "commit" command to the S3guard CLI to probe for, list and abort pending requests under
+a path, e.g. `--has-pending <path>`, `--list-pending <path>`, `--abort-pending <path>`.
+
+
+
+## The "Magic" Committer
+
+Development on this committer began before Netflix donated their committer.
+
+By making changes to the `S3AFileSystem` and the `S3ABlockOutputStream`, this committer
+manages to postpone the completion of writes of all files written to special
+("magic") directories; the final destination of the write being altered to
+that of the final job destination. When the job is committed, the pending
+writes are instantiated.
+
+With the addition of the Netflix Staging committer, the actual committer
+code now shares common formats for the persistent metadadata and shared routines
+for parallel committing of work, including all the error handling based on
+the Netflix experience.
+
+It differs in that it directly streams data to S3 (there is no staging),
+and it also stores the lists of pending commits in S3 too. That mandates
+consistent metadata on S3, which S3Guard provides.
+
+
+### Core concept: A new/modified output stream for delayed PUT commits
+
+
+This algorithm uses a modified `S3ABlockOutputStream` Output stream, which, rather
+than commit any active multipart upload in the final `close()` operation,
+it instead saves enough information into the S3 repository for an independent
+process to be able to complete or abort the upload.
+
+Originally, in `OutputStream.close()`, it chose whether to perform a single PUT or to
+complete an ongoing multipart write.
+
+If a multipart PUT is in progress, then the stream waits for the ongoing uploads
+to complete (including any final block submitted), and then builds and PUTs
+the final multipart commit operation. The list of parts (and their ordering)
+has been built up during the opt
+
+In contrast, when writing to a delayed-commit file:
+
+1. A multipart write MUST always be initiated, even for small writes. This write
+MAY be initiated during the creation of the stream.
+
+1. Instead of committing the write in the `close()` call, perform a PUT to
+a path in the S3A repository with all the information needed to commit the operation.
+That is: the final path, the multipart upload ID, and the ordered list of etags
+for the uploaded parts.
+
+
+Recognising when a file is "special" is problematic; the normal `create(Path, Boolean)`
+call must recognize when the file being created is to be a delayed-commit file,
+so returning the special new stream.
+
+
+
+This is done with a "magic" temporary directory name, `__magic`, to indicate that all files
+created under this path are not to be completed during the stream write process.
+Directories created under the path will still be created —this allows job- and
+task-specific directories to be created for individual job and task attempts.
+
+For example, the pattern `__magic/${jobID}/${taskId}` could be used to
+store pending commits to the final directory for that specific task. If that
+task is committed, all pending commit files stored in that path will be loaded
+and used to commit the final uploads.
+
+Consider a job with the final directory `/results/latest`
+
+ The intermediate directory for the task 01 attempt 01 of job `job_400_1` would be
+
+ /results/latest/__magic/job_400_1/_task_01_01
+
+This would be returned as the temp directory.
+
+When a client attempted to create the file
+`/results/latest/__magic/job_400_1/task_01_01/latest.orc.lzo` , the S3A FS would initiate
+a multipart request with the final destination of `/results/latest/latest.orc.lzo`.
+
+As data was written to the output stream, it would be incrementally uploaded as
+individual multipart PUT operations
+
+On `close()`, summary data would be written to the file
+`/results/latest/__magic/job400_1/task_01_01/latest.orc.lzo.pending`.
+This would contain the upload ID and all the parts and etags of uploaded data.
+
+
+#### Task commit
+
+The information needed to commit a task is moved from the task attempt
+to the job attempt.
+
+1. The task commit operation lists all `.pending` files in its attempt directory.
+1. The contents are loaded into a list of single pending uploads.
+1. These are merged into to a single `Pendingset` structure.
+1. Which is saved to a `.pendingset` file in the job attempt directory.
+1. Finally, the task attempt directory is deleted. In the example, this
+would be to `/results/latest/__magic/job400_1/task_01_01.pendingset`;
+
+
+A failure to load any of the single pending upload files (i.e. the file
+could not load or was considered invalid, the task is considered to
+have failed. All successfully loaded pending commits will be aborted, then
+the failure reported.
+
+Similarly, a failure to save the `.pendingset` file will trigger an
+abort of all its pending uploads.
+
+
+#### Job Commit
+
+The job committer loads all `.pendingset` files in its job attempt directory.
+
+A failure to load any of these files is considered a job failure; all
+pendingsets which could be loaded will be aborted.
+
+If all pendingsets were loaded, then every
+pending commit in the job will be committed. If any one of these commits
+failed, then all successful commits will be reverted by deleting the destination
+file.
+
+#### Supporting directory trees
+
+To allow tasks to generate data in subdirectories, a special filename `__base`
+will be used to provide an extra cue as to the final path. When mapping an output
+path `/results/latest/__magic/job_400/task_01_01/__base/2017/2017-01-01.orc.lzo.pending`
+to a final destination path, the path will become `/results/latest/2017/2017-01-01.orc.lzo`.
+That is: all directories between `__magic` and `__base` inclusive will be ignored.
+
+
+**Issues**
+
+Q. What if there are some non-`.pending` files in the task attempt directory?
+
+A. This can only happen if the magic committer is being used in an S3A client
+which does not have the "magic path" feature enabled. This will be checked for
+during job and task committer initialization.
+
+
+### Failure cases
+
+#### Network Partitioning
+
+The job/task commit protocol is expected to handle this with the task
+only committing work when the job driver tells it to. A network partition
+should trigger the task committer's cancellation of the work (this is a protcol
+above the committers).
+
+#### Job Driver failure
+
+The job will be restarted. When it completes it will delete all
+outstanding requests to the destination directory which it has not
+committed itself.
+
+#### Task failure
+
+The task will be restarted. Pending work of the task will not be committed;
+when the job driver cleans up it will cancel pending writes under the directory.
+
+#### Multiple jobs targeting the same destination directory
+
+This leaves things in an inderminate state.
+
+
+#### Failure during task commit
+
+Pending uploads will remain, but no changes will be visible.
+
+If the `.pendingset` file has been saved to the job attempt directory, the
+task has effectively committed, it has just failed to report to the
+controller. This will cause complications during job commit, as there
+may be two task PendingSet committing the same files, or committing
+files with
+
+*Proposed*: track task ID in pendingsets, recognise duplicates on load
+and then respond by cancelling one set and committing the other. (or fail?)
+
+#### Failure during job commit
+
+The destination will be left in an unknown state.
+
+#### Failure during task/job abort
+
+Failures in the abort process are not well handled in either the committers
+or indeed in the applications which use these committers. If an abort
+operation fails, what can be done?
+
+While somewhat hypothetical for the use case of a task being aborted due
+to the protocol (e.g. speculative jobs being aborted), the abort task/abort job
+calls may be made as part of the exception handling logic on a failure to commit.
+As such, the caller may assume that the abort does not fail: if it does,
+the newly thrown exception may hide the original problem.
+
+Two options present themselves
+
+1. Catch, log and swallow failures in the `abort()`
+1. Throw the exceptions, and expect the callers to handle them: review, fix
+and test that code as appropriate.
+
+Fixing the calling code does seem to be the best strategy, as it allows the
+failure to be explictly handled in the commit protocol, rather than hidden
+in the committer.::OpenFile
+
+#### Preemption
+
+Preemption is the explicit termination of work at the behest of the cluster
+scheduler. It's a failure, but a special one: pre-empted tasks must not be counted
+as a failure in any code which only allows a limited number of trackers, and the
+Job driver can assume that the task was successfully terminated.
+
+Job drivers themselves may be preempted.
+
+
+
+#### Cleaning up after complete job failure
+
+One failure case is that the entire execution framework failed; a new process
+must identify outstanding jobs with pending work, and abort them, then delete
+the appropriate `__magic` directories.
+
+This can be done either by scanning the directory tree for `__magic` directories
+and scanning underneath them, or by using the `listMultipartUploads()` call to
+list multipart uploads under a path, then cancel them. The most efficient solution
+may be to use `listMultipartUploads` to identify all outstanding request, and use that
+to identify which requests to cancel, and where to scan for `__magic` directories.
+This strategy should address scalability problems when working with repositories
+with many millions of objects —rather than list all keys searching for those
+with `/__magic/**/*.pending` in their name, work backwards from the active uploads to
+the directories with the data.
+
+We may also want to consider having a cleanup operationn in the S3 CLI to
+do the full tree scan and purge of pending items; give some statistics on
+what was found. This will keep costs down and help us identify problems
+related to cleanup.
+
+### Performance
+
+The time to upload is that of today's block upload (`s3a.fast.upload=true`)
+output stream; ongoing through the write, and in the `close()` operation,
+a delay to upload any pending data and await all outstanding uploads to complete.
+There wouldn't be any overhead of the final completion request. If no
+data had yet been uploaded, the `close()` time would be that of the initiate
+multipart request and the final put. This could perhaps be simplified by always
+requesting a multipart ID on stream creation.
+
+The time to commit each task is `O(files)`: all `.pending` files in and under the task attempt
+directory will be listed, their contents read and then an aggregate `.pendingset`
+file PUT to the job attempt directory. The `.pending` files are then deleted.
+
+The time to commit a job will be `O(files/threads)`
+
+Every `.pendingset` file in the job attempt directory must be loaded, and a PUT
+request issued for every incomplete upload listed in the files.
+
+Note that it is the bulk listing of all children which is where full consistency
+is required. If instead, the list of files to commit could be returned from
+tasks to the job committer, as the Spark commit protocol allows, it would be
+possible to commit data to an inconsistent object store.
+
+### Cost
+
+Uncommitted data in an incomplete multipart upload is billed at the storage
+cost of the S3 bucket. To keep costs down, outstanding data from
+failed jobs must be deleted. This can be done through S3 bucket lifecycle policies,
+or some command tools which we would need to write.
+
+### Limitations of this algorithm
+
+1. Files will not be visible after the `close()` call, as they will not exist.
+Any code which expected pending-commit files to be visible will fail.
+
+1. Failures of tasks and jobs will leave outstanding multipart uploads. These
+will need to be garbage collected. S3 now supports automated cleanup; S3A has
+the option to do it on startup, and we plan for the `hadoop s3` command to
+allow callers to explicitly do it. If tasks were to explicitly write the upload
+ID of writes as a write commenced, cleanup by the job committer may be possible.
+
+1. The time to write very small files may be higher than that of PUT and COPY.
+We are ignoring this problem as not relevant in production; any attempt at optimizing
+small file operations will only complicate development, maintenance and testing.
+
+1. The files containing temporary information could be mistaken for actual
+data.
+
+1. It could potentially be harder to diagnose what is causing problems. Lots of
+logging can help, especially with debug-level listing of the directory structure
+of the temporary directories.
+
+1. To reliably list all PUT requests outstanding, we need list consistency
+In the absence of a means to reliably identify when an S3 endpoint is consistent, people
+may still use eventually consistent stores, with the consequent loss of data.
+
+1. If there is more than one job simultaneously writing to the same destination
+directories, the output may get confused. This appears to hold today with the current
+commit algorithms.
+
+1. It is possible to create more than one client writing to the
+same destination file within the same S3A client/task, either sequentially or in parallel.
+
+1. Even with a consistent metadata store, if a job overwrites existing
+files, then old data may still be visible to clients reading the data, until
+the update has propagated to all replicas of the data.
+
+1. If the operation is attempting to completely overwrite the contents of
+a directory, then it is not going to work: the existing data will not be cleaned
+up. A cleanup operation would need to be included in the job commit, deleting
+all files in the destination directory which where not being overwritten.
+
+1. It requires a path element, such as `__magic` which cannot be used
+for any purpose other than for the storage of pending commit data.
+
+1. Unless extra code is added to every FS operation, it will still be possible
+to manipulate files under the `__magic` tree. That's not bad, it just potentially
+confusing.
+
+1. As written data is not materialized until the commit, it will not be possible
+for any process to read or manipulated a file which it has just created.
+
+
+
+
+### Changes to `S3ABlockOutputStream`
+
+To avoid having to copy and paste the `S3ABlockOutputStream` it has
+been modified to be constructed with a `PutTracker` class to
+managed the immediate/delayed completion of uploads.
+It will be called at appropriate points.
+
+* Initialization, returning a marker to indicate whether or not multipart
+upload is commence immediately.
+* Multipart PUT init.
+* Single put init (not used in this algorithm, but useful for completeness).
+* Block upload init, failure and completion (from the relevant thread).
+* `close()` entered; all blocks completed —returning a marker to indicate
+whether any outstanding multipart should be committed.
+* Multipart abort in `abort()` call (maybe: move core logic elsewhere).
+
+The base implementation would do nothing
+except declare that the MPU must be executed in the `close()` call.
+
+The S3A Committer version, would
+1. Always initiate a during initialization
+1. In `close()` operation save all the data required to commit later.
+
+
+## Integrating the Committers with Hadoop MapReduce
+
+
+In order to support the ubiquitous `FileOutputFormat` and subclasses,
+S3A Committers will need somehow be accepted as a valid committer by the class,
+a class which explicity expects the output committer to be `FileOutputCommitter`
+
+```java
+public Path getDefaultWorkFile(TaskAttemptContext context,
+ String extension) throws IOException{
+ PathOutputCommitter committer =
+ (PathOutputCommitter) getOutputCommitter(context);
+ return new Path(committer.getWorkPath(), getUniqueFile(context,
+ getOutputName(context), extension));
+}
+
+```
+
+Here are some options which have been considered, explored and discarded
+
+1. Adding more of a factory mechanism to create `FileOutputCommitter` instances;
+subclass this for S3A output and return it. The complexity of `FileOutputCommitter`
+and of supporting more dynamic consturction makes this dangerous from an implementation
+and maintenance perspective.
+
+1. Add a new commit algorithmm "3", which actually reads in the configured
+classname of a committer which it then instantiates and then relays the commit
+operations, passing in context information. Ths new committer interface would
+add methods for methods and attributes. This is viable, but does still change
+the existing Committer code in a way which may be high-maintenance.
+
+1. Allow the `FileOutputFormat` class to take any task/job context committer
+which implemented the `getWorkPath()` method —that being the sole
+specific feature which it needs from the `FileOutputCommitter`.
+
+
+Option 3, make `FileOutputFormat` support more generic committers, is the
+current design. It relies on the fact that the sole specific method of
+`FileOutputCommitter` which `FileOutputFormat` uses is `getWorkPath()`.
+
+This can be pulled up into a new abstract class, `PathOutputCommitter`, which
+`FileOutputCommitter` and `S3ACommitter` can implement:
+
+```java
+public abstract class PathOutputCommitter extends OutputCommitter {
+
+ /**
+ * Get the directory that the task should write results into.
+ * @return the work directory
+ */
+ public abstract Path getWorkPath() throws IOException;
+}
+```
+
+The sole change needed for `FileOutputFormat` is to change what it casts
+the context committer to:
+
+```java
+PathOutputCommitter committer =
+ (PathOutputCommitter) getOutputCommitter(context);
+```
+
+Provided that `getWorkPath()` remains the sole method which `FileOutputFormat`
+uses, these changes will allow an S3A committer to replace the `FileOutputCommitter`,
+with minimal changes to the codebase.
+
+
+Update: There is a cost to this: MRv1 API support is lost.
+
+
+### MRv1 support via `org.apache.hadoop.mapred.FileOutputFormat`
+
+A price of not subclassing `FileOutputCommitter` is that the code used
+to wrap and relay the MRv1 API calls protocol to the `FileOutputCommitter`
+will not work: the new committer will not be picked up.
+
+This is visible in Spark, where the V1 API is exported from the `RDD` class
+(`RDD.saveAsHadoopFile()`)). The successor code, `PairRDDFunctions.saveAsNewAPIHadoopFile()`
+does work: *To get high performance commits in Object Stores, the MRv2 commit protocol
+must be used, which means: the V2 classes.
+
+
+
+#### Resolved issues
+
+
+**Magic Committer: Name of directory**
+
+The design proposes the name `__magic` for the directory. HDFS and
+the various scanning routines always treat files and directories starting with `_`
+as temporary/excluded data.
+
+
+**Magic Committer: Subdirectories**
+
+It is legal to create subdirectories in a task work directory, which
+will then be moved into the destination directory, retaining that directory
+tree.
+
+That is, a if the task working dir is `dest/__magic/app1/task1/`, all files
+under `dest/__magic/app1/task1/part-0000/` must end up under the path
+`dest/part-0000/`.
+
+This behavior is relied upon for the writing of intermediate map data in an MR
+job.
+
+This means it is not simply enough to strip off all elements of under `__magic`,
+it is critical to determine the base path.
+
+Proposed: use the special name `__base` as a marker of the base element for
+committing. Under task attempts a `__base` dir is created and turned into the
+working dir. All files created under this path will be committed to the destination
+with a path relative to the base dir.
+
+More formally: the last parent element of a path which is `__base` sets the
+base for relative paths created underneath it.
+
+
+## Testing
+
+The committers can only be tested against an S3-compatible object store.
+
+Although a consistent object store is a requirement for a production deployment
+of the magic committer an inconsistent one has appeared to work during testing, simply by
+adding some delays to the operations: a task commit does not succeed until
+all the objects which it has PUT are visible in the LIST operation. Assuming
+that further listings from the same process also show the objects, the job
+committer will be able to list and commit the uploads.
+
+
+The committers have some unit tests, and integration tests based on
+the protocol integration test lifted from `org.apache.hadoop.mapreduce.lib.output.TestFileOutputCommitter`
+to test various state transitions of the commit mechanism has been extended
+to support the variants of the staging committer.
+
+There is an abstract integration test, `AbstractITCommitMRJob` which creates
+a MiniYARN cluster bonded to a MiniHDFS cluster, then submits a simple
+MR job using the relevant committer. This verifies that the committer actually
+works, rather than just "appears to follow the protocol"
+
+One feature added during this testing is that the `_SUCCESS` marker file saved is
+no-longer a 0-byte file, it is a JSON manifest file, as implemented in
+`org.apache.hadoop.fs.s3a.commit.files.SuccessData`. This file includes
+the committer used, the hostname performing the commit, timestamp data and
+a list of paths committed.
+
+```
+SuccessData{
+ committer='PartitionedStagingCommitter',
+ hostname='devbox.local',
+ description='Task committer attempt_1493832493956_0001_m_000000_0',
+ date='Wed May 03 18:28:41 BST 2017',
+ filenames=[/test/testMRJob/part-m-00000, /test/testMRJob/part-m-00002, /test/testMRJob/part-m-00001]
+}
+```
+
+This was useful a means of verifying that the correct
+committer had in fact been invoked in those forked processes: a 0-byte `_SUCCESS`
+marker implied the classic `FileOutputCommitter` had been used; if it could be read
+then it provides some details on the commit operation which are then used
+in assertions in the test suite.
+
+It has since been extended to collet metrics and other values, and has proven
+equally useful in Spark integration testing.
+
+## Integrating the Committers with Apache Spark
+
+
+Spark defines a commit protocol `org.apache.spark.internal.io.FileCommitProtocol`,
+implementing it in `HadoopMapReduceCommitProtocol` a subclass `SQLHadoopMapReduceCommitProtocol`
+which supports the configurable declaration of the underlying Hadoop committer class,
+and the `ManifestFileCommitProtocol` for Structured Streaming. The latter
+is best defined as "a complication" —but without support for it, S3 cannot be used
+as a reliable destination of stream checkpoints.
+
+One aspect of the Spark commit protocol is that alongside the Hadoop file committer,
+there's an API to request an absolute path as a target for a commit operation,
+`newTaskTempFileAbsPath(taskContext: TaskAttemptContext, absoluteDir: String, ext: String): String`;
+each task's mapping of temp-> absolute files is passed to the Spark driver
+in the `TaskCommitMessage` returned after a task performs its local
+commit operations (which includes requesting permission to commit from the executor).
+These temporary paths are renamed to the final absolute paths are renamed
+in `FileCommitProtocol.commitJob()`. This is currently a serialized rename sequence
+at the end of all other work. This use of absolute paths is used in writing
+data into a destination directory tree whose directory names is driven by
+partition names (year, month, etc).
+
+Supporting that feature is going to be challenging; either we allow each directory in the partition tree to
+have its own staging directory documenting pending PUT operations, or (better) a staging directory
+tree is built off the base path, with all pending commits tracked in a matching directory
+tree.
+
+Alternatively, the fact that Spark tasks provide data to the job committer on their
+completion means that a list of pending PUT commands could be built up, with the commit
+operations being excuted by an S3A-specific implementation of the `FileCommitProtocol`.
+As noted earlier, this may permit the reqirement for a consistent list operation
+to be bypassed. It would still be important to list what was being written, as
+it is needed to aid aborting work in failed tasks, but the list of files
+created by successful tasks could be passed directly from the task to committer,
+avoid that potentially-inconsistent list.
+
+
+#### Spark, Parquet and the Spark SQL Commit mechanism
+
+Spark's `org.apache.spark.sql.execution.datasources.parquet.ParquetFileFormat`
+Parquet output format wants a subclass of `org.apache.parquet.hadoop.ParquetOutputCommitter`,
+the option being defined by the classname in the configuration
+key `spark.sql.parquet.output.committer.class`;
+this is then patched in to the value `spark.sql.sources.outputCommitterClass`
+where it is picked up by `SQLHadoopMapReduceCommitProtocol` and instantiated
+as the committer for the work.
+
+This is presumably done so the user has the option of requesting a metadata
+summary file by setting the option `"parquet.enable.summary-metadata"`.
+Creating the summary file requires scanning every single file in the destination
+directory on the job commit, so is *very* expensive, and not something which
+we recommend when working with S3.
+
+
+To use a S3Guard committer, it must also be identified as the Parquet committer.
+The fact that instances are dynamically instantiated somewhat complicates the process.
+
+In early tests; we can switch committers for ORC output without making any changes
+to the Spark code or configuration other than configuring the factory
+for Path output committers. For Parquet support, it may be sufficient to also declare
+the classname of the specific committer (i.e not the factory).
+
+This is unfortunate as it complicates dynamically selecting a committer protocol
+based on the destination filesystem type or any per-bucket configuration.
+
+
+
+The solution as implemented in the [initial prototype](https://github.com/hortonworks-spark/cloud-integration)
+consists of two things
+
+1. A class `PathOutputCommitProtocol extends HadoopMapReduceCommitProtocol`
+which always creates the committer using the `PathOutputCommitterFactory`
+mechanism. This ensures that output format's own committers are replaced
+with an output factory mechanism.
+
+1. A class `org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter`
+which is a directly instantiable output committer that then creates a
+committer through the factory mechanism and delegates all operations to
+that committer. This allows a committer to be declared in any configuration
+option which takes a committer class, but still use the factory mechanism
+underneath.
+
+1. Add a patch to Spark 2.3 [SPARK-21762], which allows any output committer
+to be used for `ParquetFileOutput`.
+
+Overall, its a bit convoluted to implement, document and use. Users must
+declare two spark SQL options as well as three spark.hadoop ones
+
+```properties
+spark.sql.sources.commitProtocolClass=com.hortonworks.spark.cloud.commit.PathOutputCommitProtocol
+spark.sql.parquet.output.committer.class=org.apache.hadoop.mapreduce.lib.output.BindingPathOutputCommitter
+spark.hadoop.mapreduce.outputcommitter.factory.scheme.s3a=org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory
+spark.hadoop.fs.s3a.committer.name=magic
+spark.hadoop.fs.s3a.committer.magic.enabled=true
+```
+
+We could actually simplify this by adding a new algorithm "3" to the existing
+`FileOutputFormat`, telling `FileOutputFormat` itself to use the factory
+to create committers. This would then automatically pick up the factory
+Avoiding loops in this situation would be "challenging": If instantiated
+via a factory, the file committer must not attempt to use the factory
+itself.
+
+
+## Security
+
+What are the obvious possible security risks which need to be covered
+and which code reviews should check for?
+
+* Leakage of AWS credentials in jobs/logging.
+* Exposure of data to unauthorized users.
+* Exposure of workflow to unauthorized users (e.g. paths & filenames, sequence
+of queries).
+* Silent tampering of data by unauthed users.
+* Attacks on jobs which run up (large) bills by leaking pending datasets.
+* DoS attacks with malicious users sabotaging/hindering progress of other
+users' work.
+
+#### Security Risks of the Staging Committers
+
+Staged data is on local FS, in directories listed by `fs.s3a.buffer.dir`,
+falling back to the directory in `hadoop.tmp.dir`.
+These are the same directories already
+used to buffer blocks of data being written in output streams, prior to
+PUT/POST to S3. Therefore: there is more risk than before. We should
+clearly document the security aspects of the temp dirs to ensure this.
+
+As all files written by a task are not uploaded until task commit, more
+local storage is needed. A malicious user executing work on one system
+could potentially fill up the temp disk space. Mitigation: storage
+quotas in local FS, keeping temp dirs on different mounted FS from root.
+
+The intermediate `.pendingset` files are saved in HDFS under the directory in
+`fs.s3a.committer.staging.tmp.path`; defaulting to `/tmp`. This data can
+disclose the workflow (it contains the destination paths & amount of data
+generated), and if deleted, breaks the job. If malicous code were to edit
+the file, by, for example, reordering the ordered etag list, the generated
+data would be committed out of order, creating invalid files. As this is
+the (usually transient) cluster FS, any user in the cluster has the potential
+to do this.
+
+*Locking down the temporary directories is critical to prevent malicious
+cluster users damaging the workflow and output.*
+
+#### Security Risks of the Magic Committer
+
+The directory defined by `fs.s3a.buffer.dir` is used to buffer blocks
+before upload, unless the job is configured to buffer the blocks in memory.
+This is as before: no incremental risk. As blocks are deleted from the filesystem
+after upload, the amount of storage needed is determined by the data generation
+bandwidth and the data upload bandwdith.
+
+No use is made of the cluster filesystem; there are no risks there.
+
+A consistent store is required, which, for Amazon's infrastructure, means S3Guard.
+This is covered below.
+
+A malicious user with write access to the `__magic` directory could manipulate
+or delete the metadata of pending uploads, or potentially inject new work int
+the commit. Having access to the `__magic` directory implies write access
+to the parent destination directory: a malicious user could just as easily
+manipulate the final output, without needing to attack the committer's intermediate
+files.
+
+
+### Security Risks of all committers
+
+
+#### Visibility
+
+* If S3Guard is used for storing metadata, then the metadata is visible to
+all users with read access. A malicious user with write access could delete
+entries of newly generated files, so they would not be visible.
+
+
+#### Malicious Serialized Data
+
+The internal metadata summary files (`.pending` and `.pendingset`)
+could be tampered by someone malicious, and, when parsed or interpreted by
+the trusted account, used to execute untrusted code, fail, etc.
+The formats are all JSON, parsed with Jackson; we expect invalid JSON
+to result in parse errors, and fail the job. Aborting the job triggers a
+best-effort attempt to load the pending files, ignoring those which cannot
+be loaded or parsed, and aborts all pending uploads identified by the loaded
+files.
+
+* None of the strings in the parsed dataset are passed through any interpreter
+(e.g used in SQL queries, shell commands or similar). Therefore (SQL) injection
+attacks are not possible.
+
+* Some of the data *may* be printed in the log files generated during
+process execution. For example, commit Id, destination Key, etc. These
+are all treated as plain text, and should be served up as such in any browser-
+hosted view of the logs.
+
+* Some of the data is returned in the `toString()` values of the loaded
+classes. This may also be logged, observed in IDEs.
+
+* None of the content in the serialized data is displayed in any web UI.
+The vulnerability there is what if this happened in the future in some
+downstream application: would it be possible to inject Javascript into
+any of the text fields, script which could then be executed in some XSS
+attack. We may wish to consider sanitizing this data on load.
+
+* Paths in tampered data could be modified in an attempt to commit an upload across
+an existing file, or the MPU ID alterated to prematurely commit a different upload.
+These attempts will not going to succeed, because the destination
+path of the upload is declared on the initial POST to initiate the MPU, and
+operations associated with the MPU must also declare the path: if the path and
+ID are inconsistent, the operation will fail. If a valid (path, MPU) tuple
+were inserted in a tampered-with file, the commit or abort could complete or abort
+the upload prematurely. As the malicious party must already have enough access to the
+target S3 store to obtain this information, they are unlikely to need to
+tamper with the JSON files to perform such actions.
+
+
+#### Outstanding uncommitted data and its cost
+
+* Killed jobs will leak uncommitted uploads, which will run up bills.
+A restarted job will automatically purge
+all pending uploads under the destination path on job commit, so if the job
+is rerun it will cancel the pending writes of the previous job attempt.
+
+We shall also provide a CLI tool to list and delete pending uploads under a path.
+
+* configuring a store to automatically clean pending uploads after a
+time period such as 24h will guarantee that pending upload data will always
+be deleted, even without a rerun of the job or use of the CLI tool.
+
+AWS document [the permissions for MPU](http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html).
+There are special permissions `s3:ListBucketMultipartUploads`,
+`s3:ListMultipartUploadParts` and `s3:AbortMultipartUpload`. By default
+bucket owner has all these permissions, MPU initiator will be granted
+the permissions to list the upload, list hte parts and abort their own uploads.
+Bucket owner may grant/deny perms to either (this may permit a user to be
+able to initiate & complete MPU, but not delete and abort).
+
+#### Proposed security settings & recommendations
+
+* Bucket access restricted to specific IAM roles.
+* `fs.s3a.buffer.dir` set to location under `/tmp` with read & write access
+restricted to the active user.
+* `fs.s3a.committer.staging.tmp.path` should be isolated to the active
+each user. Proposed: make the default an unqualified path, `tmp/staging`,
+which will made absolute relative to the current user. In filesystems in
+which access under user's home directories are restricted, this final, absolute
+path, will not be visible to untrusted accounts.
+
+* Maybe: define the for valid characters in a text strings, and a regext for
+ validating, e,g, `[a-zA-Z0-9 \.\,\(\) \-\+]+` and then validate any free text
+ JSON fields on load and save.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/49] hadoop git commit: YARN-7332. Compute effectiveCapacity per
each resource vector. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7332. Compute effectiveCapacity per each resource vector. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0d91dfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0d91dfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0d91dfe
Branch: refs/heads/YARN-5881
Commit: f0d91dfe1813ed917ee3006d911465ad93fbcd67
Parents: c2440930
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Oct 27 10:16:33 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../scheduler/capacity/ParentQueue.java | 66 ++++++++++++--
.../scheduler/capacity/TestParentQueue.java | 94 ++++++++++++++++++++
2 files changed, 153 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d91dfe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 5ab1494..940637e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.AccessType;
@@ -68,7 +69,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -928,24 +931,25 @@ public class ParentQueue extends AbstractCSQueue {
// Factor to scale down effective resource: When cluster has sufficient
// resources, effective_min_resources will be same as configured
// min_resources.
- float effectiveMinRatio = 1;
+ Resource numeratorForMinRatio = null;
ResourceCalculator rc = this.csContext.getResourceCalculator();
if (getQueueName().equals("root")) {
if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
clusterResource, resourceByLabel, configuredMinResources)) {
- effectiveMinRatio = Resources.divide(rc, clusterResource,
- resourceByLabel, configuredMinResources);
+ numeratorForMinRatio = resourceByLabel;
}
} else {
if (Resources.lessThan(rc, clusterResource,
queueResourceQuotas.getEffectiveMinResource(label),
configuredMinResources)) {
- effectiveMinRatio = Resources.divide(rc, clusterResource,
- queueResourceQuotas.getEffectiveMinResource(label),
- configuredMinResources);
+ numeratorForMinRatio = queueResourceQuotas
+ .getEffectiveMinResource(label);
}
}
+ Map<String, Float> effectiveMinRatioPerResource = getEffectiveMinRatioPerResource(
+ configuredMinResources, numeratorForMinRatio);
+
// loop and do this for all child queues
for (CSQueue childQueue : getChildQueues()) {
Resource minResource = childQueue.getQueueResourceQuotas()
@@ -955,7 +959,8 @@ public class ParentQueue extends AbstractCSQueue {
if (childQueue.getCapacityConfigType()
.equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
- Resources.multiply(minResource, effectiveMinRatio));
+ getMinResourceNormalized(childQueue.getQueueName(), effectiveMinRatioPerResource,
+ minResource));
// Max resource of a queue should be a minimum of {configuredMaxRes,
// parentMaxRes}. parentMaxRes could be configured value. But if not
@@ -1003,6 +1008,53 @@ public class ParentQueue extends AbstractCSQueue {
}
}
+ private Resource getMinResourceNormalized(String name, Map<String, Float> effectiveMinRatio,
+ Resource minResource) {
+ Resource ret = Resource.newInstance(minResource);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation nResourceInformation = minResource
+ .getResourceInformation(i);
+
+ Float ratio = effectiveMinRatio.get(nResourceInformation.getName());
+ if (ratio != null) {
+ ret.setResourceValue(i,
+ (long) (nResourceInformation.getValue() * ratio.floatValue()));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating min resource for Queue: " + name + " as "
+ + ret.getResourceInformation(i) + ", Actual resource: "
+ + nResourceInformation.getValue() + ", ratio: "
+ + ratio.floatValue());
+ }
+ }
+ }
+ return ret;
+ }
+
+ private Map<String, Float> getEffectiveMinRatioPerResource(
+ Resource configuredMinResources, Resource numeratorForMinRatio) {
+ Map<String, Float> effectiveMinRatioPerResource = new HashMap<>();
+ if (numeratorForMinRatio != null) {
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation nResourceInformation = numeratorForMinRatio
+ .getResourceInformation(i);
+ ResourceInformation dResourceInformation = configuredMinResources
+ .getResourceInformation(i);
+
+ long nValue = nResourceInformation.getValue();
+ long dValue = UnitsConversionUtil.convert(
+ dResourceInformation.getUnits(), nResourceInformation.getUnits(),
+ dResourceInformation.getValue());
+ if (dValue != 0) {
+ effectiveMinRatioPerResource.put(nResourceInformation.getName(),
+ (float) nValue / dValue);
+ }
+ }
+ }
+ return effectiveMinRatioPerResource;
+ }
+
private void deriveCapacityFromAbsoluteConfigurations(String label,
Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d91dfe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 25a9774..fe66aba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -68,6 +68,11 @@ import org.mockito.stubbing.Answer;
public class TestParentQueue {
+ private static final Resource QUEUE_B_RESOURCE = Resource
+ .newInstance(14 * 1024, 22);
+ private static final Resource QUEUE_A_RESOURCE = Resource
+ .newInstance(6 * 1024, 10);
+
private static final Log LOG = LogFactory.getLog(TestParentQueue.class);
RMContext rmContext;
@@ -118,6 +123,23 @@ public class TestParentQueue {
LOG.info("Setup top-level queues a and b");
}
+ private void setupSingleLevelQueuesWithAbsoluteResource(
+ CapacitySchedulerConfiguration conf) {
+
+ // Define top-level queues
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B});
+
+ final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A;
+ conf.setMinimumResourceRequirement("", Q_A,
+ QUEUE_A_RESOURCE);
+
+ final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
+ conf.setMinimumResourceRequirement("", Q_B,
+ QUEUE_B_RESOURCE);
+
+ LOG.info("Setup top-level queues a and b with absolute resource");
+ }
+
private FiCaSchedulerApp getMockApplication(int appId, String user) {
FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
doReturn(user).when(application).getUser();
@@ -931,6 +953,78 @@ public class TestParentQueue {
reset(c);
}
+ @Test
+ public void testAbsoluteResourceWithChangeInClusterResource()
+ throws Exception {
+ // Setup queue configs
+ setupSingleLevelQueuesWithAbsoluteResource(csConf);
+
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
+ CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf,
+ null, CapacitySchedulerConfiguration.ROOT, queues, queues,
+ TestUtils.spyHook);
+
+ // Setup some nodes
+ final int memoryPerNode = 10;
+ int coresPerNode = 16;
+ int numNodes = 2;
+
+ Resource clusterResource = Resources.createResource(
+ numNodes * (memoryPerNode * GB), numNodes * coresPerNode);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ // Start testing
+ LeafQueue a = (LeafQueue) queues.get(A);
+ LeafQueue b = (LeafQueue) queues.get(B);
+
+ assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_B_RESOURCE);
+ assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_B_RESOURCE);
+
+ numNodes = 1;
+ clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB),
+ numNodes * coresPerNode);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ Resource QUEUE_B_RESOURCE_HALF = Resource.newInstance(7 * 1024, 11);
+ Resource QUEUE_A_RESOURCE_HALF = Resource.newInstance(3 * 1024, 5);
+ assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_B_RESOURCE);
+ assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_A_RESOURCE_HALF);
+ assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_B_RESOURCE_HALF);
+
+ coresPerNode = 40;
+ clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB),
+ numNodes * coresPerNode);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ Resource QUEUE_B_RESOURCE_70PERC = Resource.newInstance(7 * 1024, 27);
+ Resource QUEUE_A_RESOURCE_30PERC = Resource.newInstance(3 * 1024, 12);
+ assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_A_RESOURCE);
+ assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(),
+ QUEUE_B_RESOURCE);
+ assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_A_RESOURCE_30PERC);
+ assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(),
+ QUEUE_B_RESOURCE_70PERC);
+ }
+
@After
public void tearDown() throws Exception {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java
new file mode 100644
index 0000000..2c348f5
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingCommitter.java
@@ -0,0 +1,696 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.google.common.collect.Sets;
+import org.hamcrest.core.StringStartsWith;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.AWSClientIOException;
+import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants.*;
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.Paths.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*;
+import static org.apache.hadoop.test.LambdaTestUtils.*;
+
+/**
+ * The main unit test suite of the staging committer.
+ * Parameterized on thread count and unique filename policy.
+ */
+@RunWith(Parameterized.class)
+public class TestStagingCommitter extends StagingTestBase.MiniDFSTest {
+
+ private static final JobID JOB_ID = new JobID("job", 1);
+ private static final TaskAttemptID AID = new TaskAttemptID(
+ new TaskID(JOB_ID, TaskType.REDUCE, 2), 3);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestStagingCommitter.class);
+
+ private final int numThreads;
+ private final boolean uniqueFilenames;
+ private JobContext job = null;
+ private TaskAttemptContext tac = null;
+ private Configuration conf = null;
+ private MockedStagingCommitter jobCommitter = null;
+ private MockedStagingCommitter committer = null;
+
+ // created in Before
+ private S3AFileSystem mockFS = null;
+ private MockS3AFileSystem wrapperFS = null;
+
+ // created in Before
+ private StagingTestBase.ClientResults results = null;
+ private StagingTestBase.ClientErrors errors = null;
+ private AmazonS3 mockClient = null;
+ private File tmpDir;
+
+ /**
+ * Describe a test in the logs.
+ * @param text text to print
+ * @param args arguments to format in the printing
+ */
+ protected void describe(String text, Object... args) {
+ LOG.info("\n\n: {}\n", String.format(text, args));
+ }
+
+ /**
+ * Test array for parameterized test runs: how many threads and
+ * how many files to use.
+ * @return a list of parameter tuples.
+ */
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][] {
+ {0, false},
+ {1, true},
+ {3, true},
+ });
+ }
+
+ public TestStagingCommitter(int numThreads, boolean uniqueFilenames) {
+ this.numThreads = numThreads;
+ this.uniqueFilenames = uniqueFilenames;
+ }
+
+ @Before
+ public void setupCommitter() throws Exception {
+ JobConf jobConf = getConfiguration();
+ jobConf.setInt(FS_S3A_COMMITTER_THREADS, numThreads);
+ jobConf.setBoolean(FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES,
+ uniqueFilenames);
+ jobConf.set(FS_S3A_COMMITTER_STAGING_UUID,
+ UUID.randomUUID().toString());
+
+ this.results = new StagingTestBase.ClientResults();
+ this.errors = new StagingTestBase.ClientErrors();
+ this.mockClient = newMockS3Client(results, errors);
+ this.mockFS = createAndBindMockFSInstance(jobConf,
+ Pair.of(results, errors));
+ this.wrapperFS = lookupWrapperFS(jobConf);
+ // and bind the FS
+ wrapperFS.setAmazonS3Client(mockClient);
+
+
+ this.job = new JobContextImpl(jobConf, JOB_ID);
+ this.tac = new TaskAttemptContextImpl(
+ new Configuration(job.getConfiguration()), AID);
+
+ this.jobCommitter = new MockedStagingCommitter(OUTPUT_PATH, tac);
+ jobCommitter.setupJob(job);
+
+ // get the task's configuration copy so modifications take effect
+ this.conf = tac.getConfiguration();
+ this.conf.setInt(MULTIPART_SIZE, 100);
+
+ tmpDir = File.createTempFile("testStagingCommitter", "");
+ tmpDir.delete();
+ tmpDir.mkdirs();
+
+ String tmp = tmpDir.getCanonicalPath();
+ this.conf.set(BUFFER_DIR,
+ String.format("%s/local-0/, %s/local-1 ", tmp, tmp));
+
+ this.committer = new MockedStagingCommitter(OUTPUT_PATH, tac);
+ Paths.resetTempFolderCache();
+ }
+
+ @After
+ public void cleanup() {
+ try {
+ if (tmpDir != null) {
+ FileUtils.deleteDirectory(tmpDir);
+ }
+ } catch (IOException ignored) {
+
+ }
+ }
+
+ @Test
+ public void testUUIDPropagation() throws Exception {
+ Configuration config = new Configuration();
+ String jobUUID = addUUID(config);
+ assertEquals("Upload UUID", jobUUID,
+ StagingCommitter.getUploadUUID(config, JOB_ID));
+ }
+
+ private String addUUID(Configuration config) {
+ String jobUUID = UUID.randomUUID().toString();
+ config.set(FS_S3A_COMMITTER_STAGING_UUID, jobUUID);
+ return jobUUID;
+ }
+
+ @Test
+ public void testAttemptPathConstructionNoSchema() throws Exception {
+ Configuration config = new Configuration();
+ final String jobUUID = addUUID(config);
+ config.set(BUFFER_DIR, "/tmp/mr-local-0,/tmp/mr-local-1");
+ String commonPath = "file:/tmp/mr-local-";
+
+ assertThat("Missing scheme should produce local file paths",
+ getLocalTaskAttemptTempDir(config,
+ jobUUID, tac.getTaskAttemptID()).toString(),
+ StringStartsWith.startsWith(commonPath));
+ }
+
+ @Test
+ public void testAttemptPathConstructionWithSchema() throws Exception {
+ Configuration config = new Configuration();
+ final String jobUUID = addUUID(config);
+ String commonPath = "file:/tmp/mr-local-";
+
+ config.set(BUFFER_DIR,
+ "file:/tmp/mr-local-0,file:/tmp/mr-local-1");
+
+ assertThat("Path should be the same with file scheme",
+ getLocalTaskAttemptTempDir(config,
+ jobUUID, tac.getTaskAttemptID()).toString(),
+ StringStartsWith.startsWith(commonPath));
+ }
+
+ @Test
+ public void testAttemptPathConstructionWrongSchema() throws Exception {
+ Configuration config = new Configuration();
+ final String jobUUID = addUUID(config);
+ config.set(BUFFER_DIR,
+ "hdfs://nn:8020/tmp/mr-local-0,hdfs://nn:8020/tmp/mr-local-1");
+ intercept(IllegalArgumentException.class, "Wrong FS",
+ () -> getLocalTaskAttemptTempDir(config, jobUUID,
+ tac.getTaskAttemptID()));
+ }
+
+ @Test
+ public void testCommitPathConstruction() throws Exception {
+ Path committedTaskPath = committer.getCommittedTaskPath(tac);
+ assertEquals("Path should be in HDFS: " + committedTaskPath,
+ "hdfs", committedTaskPath.toUri().getScheme());
+ String ending = STAGING_UPLOADS + "/_temporary/0/task_job_0001_r_000002";
+ assertTrue("Did not end with \"" + ending +"\" :" + committedTaskPath,
+ committedTaskPath.toString().endsWith(ending));
+ }
+
+ @Test
+ public void testSingleTaskCommit() throws Exception {
+ Path file = new Path(commitTask(committer, tac, 1).iterator().next());
+
+ List<String> uploads = results.getUploads();
+ assertEquals("Should initiate one upload: " + results, 1, uploads.size());
+
+ Path committedPath = committer.getCommittedTaskPath(tac);
+ FileSystem dfs = committedPath.getFileSystem(conf);
+
+ assertEquals("Should commit to HDFS: "+ committer, getDFS(), dfs);
+
+ FileStatus[] stats = dfs.listStatus(committedPath);
+ assertEquals("Should produce one commit file: " + results, 1, stats.length);
+ assertEquals("Should name the commits file with the task ID: " + results,
+ "task_job_0001_r_000002", stats[0].getPath().getName());
+
+ PendingSet pending = PendingSet.load(dfs, stats[0].getPath());
+ assertEquals("Should have one pending commit", 1, pending.size());
+ SinglePendingCommit commit = pending.getCommits().get(0);
+ assertEquals("Should write to the correct bucket:" + results,
+ BUCKET, commit.getBucket());
+ assertEquals("Should write to the correct key: " + results,
+ OUTPUT_PREFIX + "/" + file.getName(), commit.getDestinationKey());
+
+ assertValidUpload(results.getTagsByUpload(), commit);
+ }
+
+ /**
+ * This originally verified that empty files weren't PUT. They are now.
+ * @throws Exception on a failure
+ */
+ @Test
+ public void testSingleTaskEmptyFileCommit() throws Exception {
+ committer.setupTask(tac);
+
+ Path attemptPath = committer.getTaskAttemptPath(tac);
+
+ String rand = UUID.randomUUID().toString();
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath, rand, 0);
+
+ committer.commitTask(tac);
+
+ List<String> uploads = results.getUploads();
+ assertEquals("Should initiate one upload", 1, uploads.size());
+
+ Path committedPath = committer.getCommittedTaskPath(tac);
+ FileSystem dfs = committedPath.getFileSystem(conf);
+
+ assertEquals("Should commit to HDFS", getDFS(), dfs);
+
+ assertIsFile(dfs, committedPath);
+ FileStatus[] stats = dfs.listStatus(committedPath);
+ assertEquals("Should produce one commit file", 1, stats.length);
+ assertEquals("Should name the commits file with the task ID",
+ "task_job_0001_r_000002", stats[0].getPath().getName());
+
+ PendingSet pending = PendingSet.load(dfs,
+ stats[0].getPath());
+ assertEquals("Should have one pending commit", 1, pending.size());
+ }
+
+ @Test
+ public void testSingleTaskMultiFileCommit() throws Exception {
+ int numFiles = 3;
+ Set<String> files = commitTask(committer, tac, numFiles);
+
+ List<String> uploads = results.getUploads();
+ assertEquals("Should initiate multiple uploads", numFiles, uploads.size());
+
+ Path committedPath = committer.getCommittedTaskPath(tac);
+ FileSystem dfs = committedPath.getFileSystem(conf);
+
+ assertEquals("Should commit to HDFS", getDFS(), dfs);
+ assertIsFile(dfs, committedPath);
+ FileStatus[] stats = dfs.listStatus(committedPath);
+ assertEquals("Should produce one commit file", 1, stats.length);
+ assertEquals("Should name the commits file with the task ID",
+ "task_job_0001_r_000002", stats[0].getPath().getName());
+
+ List<SinglePendingCommit> pending =
+ PendingSet.load(dfs, stats[0].getPath()).getCommits();
+ assertEquals("Should have correct number of pending commits",
+ files.size(), pending.size());
+
+ Set<String> keys = Sets.newHashSet();
+ for (SinglePendingCommit commit : pending) {
+ assertEquals("Should write to the correct bucket: " + commit,
+ BUCKET, commit.getBucket());
+ assertValidUpload(results.getTagsByUpload(), commit);
+ keys.add(commit.getDestinationKey());
+ }
+
+ assertEquals("Should write to the correct key",
+ files, keys);
+ }
+
+ @Test
+ public void testTaskInitializeFailure() throws Exception {
+ committer.setupTask(tac);
+
+ errors.failOnInit(1);
+
+ Path attemptPath = committer.getTaskAttemptPath(tac);
+ FileSystem fs = attemptPath.getFileSystem(conf);
+
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+
+ intercept(AWSClientIOException.class,
+ "Fail on init 1",
+ "Should fail during init",
+ () -> committer.commitTask(tac));
+
+ assertEquals("Should have initialized one file upload",
+ 1, results.getUploads().size());
+ assertEquals("Should abort the upload",
+ new HashSet<>(results.getUploads()),
+ getAbortedIds(results.getAborts()));
+ assertPathDoesNotExist(fs,
+ "Should remove the attempt path",
+ attemptPath);
+ }
+
+ @Test
+ public void testTaskSingleFileUploadFailure() throws Exception {
+ describe("Set up a single file upload to fail on upload 2");
+ committer.setupTask(tac);
+
+ errors.failOnUpload(2);
+
+ Path attemptPath = committer.getTaskAttemptPath(tac);
+ FileSystem fs = attemptPath.getFileSystem(conf);
+
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+
+ intercept((Class<? extends Exception>) AWSClientIOException.class,
+ "Fail on upload 2",
+ "Should fail during upload",
+ () -> {
+ committer.commitTask(tac);
+ return committer.toString();
+ });
+
+ assertEquals("Should have attempted one file upload",
+ 1, results.getUploads().size());
+ assertEquals("Should abort the upload",
+ results.getUploads().get(0),
+ results.getAborts().get(0).getUploadId());
+ assertPathDoesNotExist(fs, "Should remove the attempt path",
+ attemptPath);
+ }
+
+ @Test
+ public void testTaskMultiFileUploadFailure() throws Exception {
+ committer.setupTask(tac);
+
+ errors.failOnUpload(5);
+
+ Path attemptPath = committer.getTaskAttemptPath(tac);
+ FileSystem fs = attemptPath.getFileSystem(conf);
+
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+
+ intercept((Class<? extends Exception>) AWSClientIOException.class,
+ "Fail on upload 5",
+ "Should fail during upload",
+ () -> {
+ committer.commitTask(tac);
+ return committer.toString();
+ });
+
+ assertEquals("Should have attempted two file uploads",
+ 2, results.getUploads().size());
+ assertEquals("Should abort the upload",
+ new HashSet<>(results.getUploads()),
+ getAbortedIds(results.getAborts()));
+ assertPathDoesNotExist(fs, "Should remove the attempt path",
+ attemptPath);
+ }
+
+ @Test
+ public void testTaskUploadAndAbortFailure() throws Exception {
+ committer.setupTask(tac);
+
+ errors.failOnUpload(5);
+ errors.failOnAbort(0);
+
+ Path attemptPath = committer.getTaskAttemptPath(tac);
+ FileSystem fs = attemptPath.getFileSystem(conf);
+
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+ writeOutputFile(tac.getTaskAttemptID(), attemptPath,
+ UUID.randomUUID().toString(), 10);
+
+ intercept((Class<? extends Exception>) AWSClientIOException.class,
+ "Fail on upload 5",
+ "Should suppress abort failure, propagate upload failure",
+ ()-> {
+ committer.commitTask(tac);
+ return committer.toString();
+ });
+
+ assertEquals("Should have attempted two file uploads",
+ 2, results.getUploads().size());
+ assertEquals("Should not have succeeded with any aborts",
+ new HashSet<>(),
+ getAbortedIds(results.getAborts()));
+ assertPathDoesNotExist(fs, "Should remove the attempt path", attemptPath);
+ }
+
+ @Test
+ public void testSingleTaskAbort() throws Exception {
+ committer.setupTask(tac);
+
+ Path attemptPath = committer.getTaskAttemptPath(tac);
+ FileSystem fs = attemptPath.getFileSystem(conf);
+
+ Path outPath = writeOutputFile(
+ tac.getTaskAttemptID(), attemptPath, UUID.randomUUID().toString(), 10);
+
+ committer.abortTask(tac);
+
+ assertEquals("Should not upload anything",
+ 0, results.getUploads().size());
+ assertEquals("Should not upload anything",
+ 0, results.getParts().size());
+ assertPathDoesNotExist(fs, "Should remove all attempt data", outPath);
+ assertPathDoesNotExist(fs, "Should remove the attempt path", attemptPath);
+
+ }
+
+ @Test
+ public void testJobCommit() throws Exception {
+ Path jobAttemptPath = jobCommitter.getJobAttemptPath(job);
+ FileSystem fs = jobAttemptPath.getFileSystem(conf);
+
+ Set<String> uploads = runTasks(job, 4, 3);
+ assertNotEquals(0, uploads.size());
+
+ assertPathExists(fs, "No job attempt path", jobAttemptPath);
+
+ jobCommitter.commitJob(job);
+ assertEquals("Should have aborted no uploads",
+ 0, results.getAborts().size());
+
+ assertEquals("Should have deleted no uploads",
+ 0, results.getDeletes().size());
+
+ assertEquals("Should have committed all uploads",
+ uploads, getCommittedIds(results.getCommits()));
+
+ assertPathDoesNotExist(fs, "jobAttemptPath not deleted", jobAttemptPath);
+
+ }
+
+ @Test
+ public void testJobCommitFailure() throws Exception {
+ Path jobAttemptPath = jobCommitter.getJobAttemptPath(job);
+ FileSystem fs = jobAttemptPath.getFileSystem(conf);
+
+ Set<String> uploads = runTasks(job, 4, 3);
+
+ assertPathExists(fs, "No job attempt path", jobAttemptPath);
+
+ errors.failOnCommit(5);
+ setMockLogLevel(MockS3AFileSystem.LOG_NAME);
+
+ intercept(AWSClientIOException.class,
+ "Fail on commit 5",
+ "Should propagate the commit failure",
+ () -> {
+ jobCommitter.commitJob(job);
+ return jobCommitter.toString();
+ });
+
+ assertEquals("Should have succeeded to commit some uploads",
+ 5, results.getCommits().size());
+
+ assertEquals("Should have deleted the files that succeeded",
+ 5, results.getDeletes().size());
+
+ Set<String> commits = results.getCommits()
+ .stream()
+ .map((commit) -> commit.getBucketName() + commit.getKey())
+ .collect(Collectors.toSet());
+
+ Set<String> deletes = results.getDeletes()
+ .stream()
+ .map((delete) -> delete.getBucketName() + delete.getKey())
+ .collect(Collectors.toSet());
+
+ assertEquals("Committed and deleted objects should match",
+ commits, deletes);
+
+ assertEquals("Mismatch in aborted upload count",
+ 7, results.getAborts().size());
+
+ Set<String> uploadIds = getCommittedIds(results.getCommits());
+ uploadIds.addAll(getAbortedIds(results.getAborts()));
+
+ assertEquals("Should have committed/deleted or aborted all uploads",
+ uploads, uploadIds);
+
+ assertPathDoesNotExist(fs, "jobAttemptPath not deleted", jobAttemptPath);
+ }
+
+ @Test
+ public void testJobAbort() throws Exception {
+ Path jobAttemptPath = jobCommitter.getJobAttemptPath(job);
+ FileSystem fs = jobAttemptPath.getFileSystem(conf);
+
+ Set<String> uploads = runTasks(job, 4, 3);
+
+ assertPathExists(fs, "No job attempt path", jobAttemptPath);
+ jobCommitter.abortJob(job, JobStatus.State.KILLED);
+ assertEquals("Should have committed no uploads: " + jobCommitter,
+ 0, results.getCommits().size());
+
+ assertEquals("Should have deleted no uploads: " + jobCommitter,
+ 0, results.getDeletes().size());
+
+ assertEquals("Should have aborted all uploads: " + jobCommitter,
+ uploads, getAbortedIds(results.getAborts()));
+
+ assertPathDoesNotExist(fs, "jobAttemptPath not deleted", jobAttemptPath);
+ }
+
+ /**
+ * Run tasks, return the uploaded dataset. The upload data is
+ * extracted from the {@link #results} field; this is reset
+ * before the operation.
+ * @param jobContext job ctx
+ * @param numTasks number of tasks to run
+ * @param numFiles number of files for each task to generate
+ * @return a set of all uploads
+ * @throws IOException on a failure.
+ */
+ private Set<String> runTasks(JobContext jobContext,
+ int numTasks, int numFiles)
+ throws IOException {
+ results.resetUploads();
+ Set<String> uploads = Sets.newHashSet();
+
+ for (int taskId = 0; taskId < numTasks; taskId += 1) {
+ TaskAttemptID attemptID = new TaskAttemptID(
+ new TaskID(JOB_ID, TaskType.REDUCE, taskId),
+ (taskId * 37) % numTasks);
+ TaskAttemptContext attempt = new TaskAttemptContextImpl(
+ new Configuration(jobContext.getConfiguration()), attemptID);
+ MockedStagingCommitter taskCommitter = new MockedStagingCommitter(
+ OUTPUT_PATH, attempt);
+ commitTask(taskCommitter, attempt, numFiles);
+ }
+
+ uploads.addAll(results.getUploads());
+ return uploads;
+ }
+
+ private static Set<String> getAbortedIds(
+ List<AbortMultipartUploadRequest> aborts) {
+ return aborts.stream()
+ .map(AbortMultipartUploadRequest::getUploadId)
+ .collect(Collectors.toSet());
+ }
+
+ private static Set<String> getCommittedIds(
+ List<CompleteMultipartUploadRequest> commits) {
+ return commits.stream()
+ .map(CompleteMultipartUploadRequest::getUploadId)
+ .collect(Collectors.toSet());
+ }
+
+ private Set<String> commitTask(StagingCommitter staging,
+ TaskAttemptContext attempt,
+ int numFiles)
+ throws IOException {
+ Path attemptPath = staging.getTaskAttemptPath(attempt);
+
+ Set<String> files = Sets.newHashSet();
+ for (int i = 0; i < numFiles; i += 1) {
+ Path outPath = writeOutputFile(
+ attempt.getTaskAttemptID(), attemptPath, UUID.randomUUID().toString(),
+ 10 * (i + 1));
+ files.add(OUTPUT_PREFIX +
+ "/" + outPath.getName()
+ + (uniqueFilenames ? ("-" + staging.getUUID()) : ""));
+ }
+
+ staging.commitTask(attempt);
+
+ return files;
+ }
+
+ private static void assertValidUpload(Map<String, List<String>> parts,
+ SinglePendingCommit commit) {
+ assertTrue("Should commit a valid uploadId",
+ parts.containsKey(commit.getUploadId()));
+
+ List<String> tags = parts.get(commit.getUploadId());
+ assertEquals("Should commit the correct number of file parts",
+ tags.size(), commit.getPartCount());
+
+ for (int i = 0; i < tags.size(); i += 1) {
+ assertEquals("Should commit the correct part tags",
+ tags.get(i), commit.getEtags().get(i));
+ }
+ }
+
+ private static Path writeOutputFile(TaskAttemptID id, Path dest,
+ String content, long copies)
+ throws IOException {
+ String fileName = ((id.getTaskType() == TaskType.REDUCE) ? "r_" : "m_") +
+ id.getTaskID().getId() + "_" + id.getId() + "_" +
+ UUID.randomUUID().toString();
+ Path outPath = new Path(dest, fileName);
+ FileSystem fs = outPath.getFileSystem(getConfiguration());
+
+ try (OutputStream out = fs.create(outPath)) {
+ byte[] bytes = content.getBytes(StandardCharsets.UTF_8);
+ for (int i = 0; i < copies; i += 1) {
+ out.write(bytes);
+ }
+ }
+
+ return outPath;
+ }
+
+ /**
+ * Used during debugging mock test failures; cranks up logging of method
+ * calls.
+ * @param level log level
+ */
+ private void setMockLogLevel(int level) {
+ wrapperFS.setLogEvents(level);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
new file mode 100644
index 0000000..4f0189e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.*;
+
+/** Mocking test of directory committer. */
+public class TestStagingDirectoryOutputCommitter
+ extends StagingTestBase.JobCommitterTest<DirectoryStagingCommitter> {
+
+ @Override
+ DirectoryStagingCommitter newJobCommitter() throws Exception {
+ return new DirectoryStagingCommitter(OUTPUT_PATH,
+ createTaskAttemptForJob());
+ }
+
+ @Test
+ public void testBadConflictMode() throws Throwable {
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, "merge");
+ intercept(IllegalArgumentException.class,
+ "MERGE", "committer conflict",
+ this::newJobCommitter);
+ }
+
+ @Test
+ public void testDefaultConflictResolution() throws Exception {
+ getJob().getConfiguration().unset(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE);
+ verifyFailureConflictOutcome();
+ }
+ @Test
+ public void testFailConflictResolution() throws Exception {
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_FAIL);
+ verifyFailureConflictOutcome();
+ }
+
+ protected void verifyFailureConflictOutcome() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+ pathExists(mockS3, OUTPUT_PATH);
+ final DirectoryStagingCommitter committer = newJobCommitter();
+
+ intercept(PathExistsException.class,
+ InternalCommitterConstants.E_DEST_EXISTS,
+ "Should throw an exception because the path exists",
+ () -> committer.setupJob(getJob()));
+
+ intercept(PathExistsException.class,
+ InternalCommitterConstants.E_DEST_EXISTS,
+ "Should throw an exception because the path exists",
+ () -> committer.commitJob(getJob()));
+
+ reset(mockS3);
+ pathDoesNotExist(mockS3, OUTPUT_PATH);
+
+ committer.setupJob(getJob());
+ verifyExistenceChecked(mockS3, OUTPUT_PATH);
+ verifyNoMoreInteractions(mockS3);
+
+ reset(mockS3);
+ pathDoesNotExist(mockS3, OUTPUT_PATH);
+ committer.commitJob(getJob());
+ verifyExistenceChecked(mockS3, OUTPUT_PATH);
+ verifyCompletion(mockS3);
+ }
+
+ @Test
+ public void testAppendConflictResolution() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ pathExists(mockS3, OUTPUT_PATH);
+
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
+
+ final DirectoryStagingCommitter committer = newJobCommitter();
+
+ committer.setupJob(getJob());
+ verifyNoMoreInteractions(mockS3);
+
+ Mockito.reset(mockS3);
+ pathExists(mockS3, OUTPUT_PATH);
+
+ committer.commitJob(getJob());
+ verifyCompletion(mockS3);
+ }
+
+ @Test
+ public void testReplaceConflictResolution() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ pathExists(mockS3, OUTPUT_PATH);
+
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE);
+
+ final DirectoryStagingCommitter committer = newJobCommitter();
+
+ committer.setupJob(getJob());
+ verifyNoMoreInteractions(mockS3);
+
+ Mockito.reset(mockS3);
+ pathExists(mockS3, OUTPUT_PATH);
+ canDelete(mockS3, OUTPUT_PATH);
+
+ committer.commitJob(getJob());
+ verifyDeleted(mockS3, OUTPUT_PATH);
+ verifyCompletion(mockS3);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java
new file mode 100644
index 0000000..139b4e3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedFileListing.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Lists;
+import org.junit.After;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*;
+import static org.hamcrest.CoreMatchers.allOf;
+import static org.hamcrest.CoreMatchers.hasItem;
+
+/**
+ * Test partitioned staging committer's logic for putting data in the right
+ * place.
+ */
+public class TestStagingPartitionedFileListing
+ extends TaskCommitterTest<PartitionedStagingCommitter> {
+
+ @Override
+ PartitionedStagingCommitter newJobCommitter() throws IOException {
+ return new PartitionedStagingCommitter(OUTPUT_PATH,
+ createTaskAttemptForJob());
+ }
+
+ @Override
+ PartitionedStagingCommitter newTaskCommitter() throws IOException {
+ return new PartitionedStagingCommitter(OUTPUT_PATH, getTAC());
+ }
+
+ private FileSystem attemptFS;
+ private Path attemptPath;
+
+ @After
+ public void cleanupAttempt() {
+ cleanup("teardown", attemptFS, attemptPath);
+ }
+
+ @Test
+ public void testTaskOutputListing() throws Exception {
+ PartitionedStagingCommitter committer = newTaskCommitter();
+
+ // create files in the attempt path that should be found by getTaskOutput
+ attemptPath = committer.getTaskAttemptPath(getTAC());
+ attemptFS = attemptPath.getFileSystem(
+ getTAC().getConfiguration());
+ attemptFS.delete(attemptPath, true);
+
+ try {
+ List<String> expectedFiles = Lists.newArrayList();
+ for (String dateint : Arrays.asList("20161115", "20161116")) {
+ for (String hour : Arrays.asList("13", "14")) {
+ String relative = "dateint=" + dateint + "/hour=" + hour +
+ "/" + UUID.randomUUID().toString() + ".parquet";
+ expectedFiles.add(relative);
+ attemptFS.create(new Path(attemptPath, relative)).close();
+ }
+ }
+
+ List<String> actualFiles = committer.getTaskOutput(getTAC())
+ .stream()
+ .map(stat -> Paths.getRelativePath(attemptPath,
+ stat.getPath()))
+ .collect(Collectors.toList());
+ Collections.sort(expectedFiles);
+ Collections.sort(actualFiles);
+ assertEquals("File sets should match", expectedFiles, actualFiles);
+ } finally {
+ deleteQuietly(attemptFS, attemptPath, true);
+ }
+
+ }
+
+ @Test
+ public void testTaskOutputListingWithHiddenFiles() throws Exception {
+ PartitionedStagingCommitter committer = newTaskCommitter();
+
+ // create files in the attempt path that should be found by getTaskOutput
+ attemptPath = committer.getTaskAttemptPath(getTAC());
+ attemptFS = attemptPath.getFileSystem(
+ getTAC().getConfiguration());
+ attemptFS.delete(attemptPath, true);
+
+ try {
+ List<String> expectedFiles = Lists.newArrayList();
+ for (String dateint : Arrays.asList("20161115", "20161116")) {
+ String metadata = "dateint=" + dateint + "/" + "_metadata";
+ attemptFS.create(new Path(attemptPath, metadata)).close();
+
+ for (String hour : Arrays.asList("13", "14")) {
+ String relative = "dateint=" + dateint + "/hour=" + hour +
+ "/" + UUID.randomUUID().toString() + ".parquet";
+ expectedFiles.add(relative);
+ attemptFS.create(new Path(attemptPath, relative)).close();
+
+ String partial = "dateint=" + dateint + "/hour=" + hour +
+ "/." + UUID.randomUUID().toString() + ".partial";
+ attemptFS.create(new Path(attemptPath, partial)).close();
+ }
+ }
+
+ List<String> actualFiles = committer.getTaskOutput(getTAC()).stream()
+ .map(stat -> Paths.getRelativePath(attemptPath, stat.getPath()))
+ .collect(Collectors.toList());
+ Collections.sort(expectedFiles);
+ Collections.sort(actualFiles);
+ assertEquals("File sets should match", expectedFiles, actualFiles);
+ } finally {
+ deleteQuietly(attemptFS, attemptPath, true);
+ }
+
+ }
+
+ @Test
+ public void testPartitionsResolution() throws Throwable {
+
+ File tempDir = getTempDir();
+ File partitionsDir = new File(tempDir, "partitions");
+
+ attemptPath = new Path(partitionsDir.toURI());
+ attemptFS = FileSystem.getLocal(getJob().getConfiguration());
+ deleteQuietly(attemptFS, attemptPath, true);
+ attemptFS.mkdirs(attemptPath);
+ // initial partitioning -> empty
+ assertTrue(Paths.getPartitions(attemptPath, new ArrayList<>(0)).isEmpty());
+ String oct2017 = "year=2017/month=10";
+ Path octLog = new Path(attemptPath, oct2017 + "/log-2017-10-04.txt");
+ touch(attemptFS, octLog);
+ assertThat(listPartitions(attemptFS, attemptPath), hasItem(oct2017));
+
+ // add a root entry and it ends up under the table_root entry
+ Path rootFile = new Path(attemptPath, "root.txt");
+ touch(attemptFS, rootFile);
+ assertThat(listPartitions(attemptFS, attemptPath),
+ allOf(hasItem(oct2017),
+ hasItem(StagingCommitterConstants.TABLE_ROOT)));
+ }
+
+ /**
+ * List files in a filesystem using {@code listFiles()},
+ * then get all the partitions.
+ * @param fs filesystem
+ * @param base base of tree
+ * @return a list of partitions
+ * @throws IOException failure
+ */
+ private Set<String> listPartitions(FileSystem fs, Path base)
+ throws IOException {
+ List<FileStatus> statusList = mapLocatedFiles(
+ fs.listFiles(base, true), s -> (FileStatus) s);
+ return Paths.getPartitions(base, statusList);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java
new file mode 100644
index 0000000..4df3912
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedJobCommit.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+import com.google.common.collect.Lists;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.PathCommitException;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/** Mocking test of partitioned committer. */
+public class TestStagingPartitionedJobCommit
+ extends StagingTestBase.JobCommitterTest<PartitionedStagingCommitter> {
+
+ @Override
+ public void setupJob() throws Exception {
+ super.setupJob();
+ getWrapperFS().setLogEvents(MockS3AFileSystem.LOG_NAME);
+ }
+
+ @Override
+ PartitionedStagingCommitter newJobCommitter() throws IOException {
+ return new PartitionedStagingCommitterForTesting(createTaskAttemptForJob());
+ }
+
+ /**
+ * Subclass of the Partitioned Staging committer used in the test cases.
+ */
+ private static final class PartitionedStagingCommitterForTesting
+ extends PartitionedCommitterForTesting {
+
+ private boolean aborted = false;
+
+ private PartitionedStagingCommitterForTesting(TaskAttemptContext context)
+ throws IOException {
+ super(OUTPUT_PATH, context);
+ }
+
+ @Override
+ protected List<SinglePendingCommit> listPendingUploadsToCommit(
+ JobContext context) throws IOException {
+ List<SinglePendingCommit> pending = Lists.newArrayList();
+
+ for (String dateint : Arrays.asList("20161115", "20161116")) {
+ for (String hour : Arrays.asList("13", "14")) {
+ String key = OUTPUT_PREFIX + "/dateint=" + dateint + "/hour=" + hour +
+ "/" + UUID.randomUUID().toString() + ".parquet";
+ SinglePendingCommit commit = new SinglePendingCommit();
+ commit.setBucket(BUCKET);
+ commit.setDestinationKey(key);
+ commit.setUri("s3a://" + BUCKET + "/" + key);
+ commit.setUploadId(UUID.randomUUID().toString());
+ commit.setEtags(new ArrayList<>());
+ pending.add(commit);
+ }
+ }
+ return pending;
+ }
+
+ @Override
+ protected void abortJobInternal(JobContext context,
+ boolean suppressExceptions) throws IOException {
+ this.aborted = true;
+ super.abortJobInternal(context, suppressExceptions);
+ }
+ }
+
+ @Test
+ public void testDefaultFailAndAppend() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ // both fail and append don't check. fail is enforced at the task level.
+ for (String mode : Arrays.asList(null, CONFLICT_MODE_FAIL,
+ CONFLICT_MODE_APPEND)) {
+ if (mode != null) {
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, mode);
+ } else {
+ getJob().getConfiguration().unset(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE);
+ }
+
+ PartitionedStagingCommitter committer = newJobCommitter();
+
+ // no directories exist
+ committer.commitJob(getJob());
+
+ // parent and peer directories exist
+ reset(mockS3);
+ pathsExist(mockS3, "dateint=20161116",
+ "dateint=20161116/hour=10");
+ committer.commitJob(getJob());
+ verifyCompletion(mockS3);
+
+ // a leaf directory exists.
+ // NOTE: this is not checked during job commit, the commit succeeds.
+ reset(mockS3);
+ pathsExist(mockS3, "dateint=20161115/hour=14");
+ committer.commitJob(getJob());
+ verifyCompletion(mockS3);
+ }
+ }
+
+ @Test
+ public void testBadConflictMode() throws Throwable {
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, "merge");
+ intercept(IllegalArgumentException.class,
+ "MERGE", "committer conflict",
+ this::newJobCommitter);
+ }
+
+ @Test
+ public void testReplace() throws Exception {
+ S3AFileSystem mockS3 = getMockS3A();
+
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE);
+
+ PartitionedStagingCommitter committer = newJobCommitter();
+
+ committer.commitJob(getJob());
+ verifyReplaceCommitActions(mockS3);
+ verifyCompletion(mockS3);
+
+ // parent and peer directories exist
+ reset(mockS3);
+ pathsExist(mockS3, "dateint=20161115",
+ "dateint=20161115/hour=12");
+
+ committer.commitJob(getJob());
+ verifyReplaceCommitActions(mockS3);
+ verifyCompletion(mockS3);
+
+ // partition directories exist and should be removed
+ reset(mockS3);
+ pathsExist(mockS3, "dateint=20161115/hour=12",
+ "dateint=20161115/hour=13");
+ canDelete(mockS3, "dateint=20161115/hour=13");
+
+ committer.commitJob(getJob());
+ verifyDeleted(mockS3, "dateint=20161115/hour=13");
+ verifyReplaceCommitActions(mockS3);
+ verifyCompletion(mockS3);
+
+ // partition directories exist and should be removed
+ reset(mockS3);
+ pathsExist(mockS3, "dateint=20161116/hour=13",
+ "dateint=20161116/hour=14");
+
+ canDelete(mockS3, "dateint=20161116/hour=13",
+ "dateint=20161116/hour=14");
+
+ committer.commitJob(getJob());
+ verifyReplaceCommitActions(mockS3);
+ verifyDeleted(mockS3, "dateint=20161116/hour=13");
+ verifyDeleted(mockS3, "dateint=20161116/hour=14");
+ verifyCompletion(mockS3);
+ }
+
+
+ /**
+ * Verify the actions which replace does, essentially: delete the parent
+ * partitions.
+ * @param mockS3 s3 mock
+ */
+ protected void verifyReplaceCommitActions(FileSystem mockS3)
+ throws IOException {
+ verifyDeleted(mockS3, "dateint=20161115/hour=13");
+ verifyDeleted(mockS3, "dateint=20161115/hour=14");
+ verifyDeleted(mockS3, "dateint=20161116/hour=13");
+ verifyDeleted(mockS3, "dateint=20161116/hour=14");
+ }
+
+ @Test
+ public void testReplaceWithDeleteFailure() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE);
+
+ final PartitionedStagingCommitter committer = newJobCommitter();
+
+ pathsExist(mockS3, "dateint=20161116/hour=14");
+ when(mockS3
+ .delete(
+ new Path(OUTPUT_PATH, "dateint=20161116/hour=14"),
+ true))
+ .thenThrow(new PathCommitException("fake",
+ "Fake IOException for delete"));
+
+ intercept(PathCommitException.class, "Fake IOException for delete",
+ "Should throw the fake IOException",
+ () -> committer.commitJob(getJob()));
+
+ verifyReplaceCommitActions(mockS3);
+ verifyDeleted(mockS3, "dateint=20161116/hour=14");
+ assertTrue("Should have aborted",
+ ((PartitionedStagingCommitterForTesting) committer).aborted);
+ verifyCompletion(mockS3);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java
new file mode 100644
index 0000000..ddcb56e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingPartitionedTaskCommit.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.mapreduce.JobContext;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+import static org.mockito.Mockito.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.*;
+
+/** Mocking test of the partitioned committer. */
+public class TestStagingPartitionedTaskCommit
+ extends StagingTestBase.TaskCommitterTest<PartitionedStagingCommitter> {
+
+ @Override
+ PartitionedStagingCommitter newJobCommitter() throws IOException {
+ return new PartitionedStagingCommitter(OUTPUT_PATH,
+ createTaskAttemptForJob());
+ }
+
+ @Override
+ PartitionedStagingCommitter newTaskCommitter() throws Exception {
+ return new PartitionedStagingCommitter(OUTPUT_PATH, getTAC());
+ }
+
+ // The set of files used by this test
+ private static List<String> relativeFiles = Lists.newArrayList();
+
+ @BeforeClass
+ public static void createRelativeFileList() {
+ for (String dateint : Arrays.asList("20161115", "20161116")) {
+ for (String hour : Arrays.asList("14", "15")) {
+ String relative = "dateint=" + dateint + "/hour=" + hour +
+ "/" + UUID.randomUUID().toString() + ".parquet";
+ relativeFiles.add(relative);
+ }
+ }
+ }
+
+ @Test
+ public void testBadConflictMode() throws Throwable {
+ getJob().getConfiguration().set(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, "merge");
+ intercept(IllegalArgumentException.class,
+ "MERGE", "committer conflict", this::newJobCommitter);
+ }
+
+ @Test
+ public void testDefault() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ JobContext job = getJob();
+ job.getConfiguration().unset(
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE);
+ final PartitionedStagingCommitter committer = newTaskCommitter();
+
+ committer.setupTask(getTAC());
+ assertConflictResolution(committer, job, ConflictResolution.FAIL);
+ createTestOutputFiles(relativeFiles,
+ committer.getTaskAttemptPath(getTAC()), getTAC().getConfiguration());
+
+ // test failure when one partition already exists
+ reset(mockS3);
+ pathExists(mockS3, new Path(OUTPUT_PATH, relativeFiles.get(0)).getParent());
+
+ intercept(PathExistsException.class,
+ InternalCommitterConstants.E_DEST_EXISTS,
+ "Expected a PathExistsException as a partition already exists",
+ () -> committer.commitTask(getTAC()));
+
+ // test success
+ reset(mockS3);
+
+ committer.commitTask(getTAC());
+ Set<String> files = Sets.newHashSet();
+ for (InitiateMultipartUploadRequest request :
+ getMockResults().getRequests().values()) {
+ assertEquals(BUCKET, request.getBucketName());
+ files.add(request.getKey());
+ }
+ assertEquals("Should have the right number of uploads",
+ relativeFiles.size(), files.size());
+
+ Set<String> expected = buildExpectedList(committer);
+
+ assertEquals("Should have correct paths", expected, files);
+ }
+
+ @Test
+ public void testFail() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ getTAC().getConfiguration()
+ .set(FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_FAIL);
+
+ final PartitionedStagingCommitter committer = newTaskCommitter();
+
+ committer.setupTask(getTAC());
+ createTestOutputFiles(relativeFiles,
+ committer.getTaskAttemptPath(getTAC()), getTAC().getConfiguration());
+
+ // test failure when one partition already exists
+ reset(mockS3);
+ pathExists(mockS3, new Path(OUTPUT_PATH, relativeFiles.get(1)).getParent());
+
+ intercept(PathExistsException.class, "",
+ "Should complain because a partition already exists",
+ () -> committer.commitTask(getTAC()));
+
+ // test success
+ reset(mockS3);
+
+ committer.commitTask(getTAC());
+ Set<String> files = Sets.newHashSet();
+ for (InitiateMultipartUploadRequest request :
+ getMockResults().getRequests().values()) {
+ assertEquals(BUCKET, request.getBucketName());
+ files.add(request.getKey());
+ }
+ assertEquals("Should have the right number of uploads",
+ relativeFiles.size(), files.size());
+
+ Set<String> expected = buildExpectedList(committer);
+
+ assertEquals("Should have correct paths", expected, files);
+ }
+
+ @Test
+ public void testAppend() throws Exception {
+ FileSystem mockS3 = getMockS3A();
+
+ getTAC().getConfiguration()
+ .set(FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_APPEND);
+
+ PartitionedStagingCommitter committer = newTaskCommitter();
+
+ committer.setupTask(getTAC());
+ createTestOutputFiles(relativeFiles,
+ committer.getTaskAttemptPath(getTAC()), getTAC().getConfiguration());
+
+ // test success when one partition already exists
+ reset(mockS3);
+ pathExists(mockS3, new Path(OUTPUT_PATH, relativeFiles.get(2)).getParent());
+
+ committer.commitTask(getTAC());
+ Set<String> files = Sets.newHashSet();
+ for (InitiateMultipartUploadRequest request :
+ getMockResults().getRequests().values()) {
+ assertEquals(BUCKET, request.getBucketName());
+ files.add(request.getKey());
+ }
+ assertEquals("Should have the right number of uploads",
+ relativeFiles.size(), files.size());
+
+ Set<String> expected = buildExpectedList(committer);
+
+ assertEquals("Should have correct paths", expected, files);
+ }
+
+ @Test
+ public void testReplace() throws Exception {
+ // TODO: this committer needs to delete the data that already exists
+ // This test should assert that the delete was done
+ FileSystem mockS3 = getMockS3A();
+
+ getTAC().getConfiguration()
+ .set(FS_S3A_COMMITTER_STAGING_CONFLICT_MODE, CONFLICT_MODE_REPLACE);
+
+ PartitionedStagingCommitter committer = newTaskCommitter();
+
+ committer.setupTask(getTAC());
+ createTestOutputFiles(relativeFiles,
+ committer.getTaskAttemptPath(getTAC()), getTAC().getConfiguration());
+
+ // test success when one partition already exists
+ reset(mockS3);
+ pathExists(mockS3, new Path(OUTPUT_PATH, relativeFiles.get(3)).getParent());
+
+ committer.commitTask(getTAC());
+ Set<String> files = Sets.newHashSet();
+ for (InitiateMultipartUploadRequest request :
+ getMockResults().getRequests().values()) {
+ assertEquals(BUCKET, request.getBucketName());
+ files.add(request.getKey());
+ }
+ assertEquals("Should have the right number of uploads",
+ relativeFiles.size(), files.size());
+
+ Set<String> expected = buildExpectedList(committer);
+
+ assertEquals("Should have correct paths", expected, files);
+ }
+
+ public Set<String> buildExpectedList(StagingCommitter committer) {
+ Set<String> expected = Sets.newHashSet();
+ boolean unique = committer.useUniqueFilenames();
+ for (String relative : relativeFiles) {
+ expected.add(OUTPUT_PREFIX +
+ "/" +
+ (unique ? Paths.addUUID(relative, committer.getUUID()) : relative));
+ }
+ return expected;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITDirectoryCommitMRJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITDirectoryCommitMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITDirectoryCommitMRJob.java
new file mode 100644
index 0000000..c10ebed
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITDirectoryCommitMRJob.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging.integration;
+
+import org.apache.hadoop.fs.s3a.commit.AbstractITCommitMRJob;
+import org.apache.hadoop.fs.s3a.commit.staging.DirectoryStagingCommitter;
+
+/**
+ * Full integration test for the directory committer.
+ */
+public class ITDirectoryCommitMRJob extends AbstractITCommitMRJob {
+
+ @Override
+ protected String committerName() {
+ return DirectoryStagingCommitter.NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITPartitionCommitMRJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITPartitionCommitMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITPartitionCommitMRJob.java
new file mode 100644
index 0000000..1c19a95
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITPartitionCommitMRJob.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging.integration;
+
+import org.apache.hadoop.fs.s3a.commit.AbstractITCommitMRJob;
+import org.apache.hadoop.fs.s3a.commit.staging.PartitionedStagingCommitter;
+
+/**
+ * Full integration test for the partition committer.
+ */
+public class ITPartitionCommitMRJob extends AbstractITCommitMRJob {
+
+ @Override
+ protected String committerName() {
+ return PartitionedStagingCommitter.NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITStagingCommitMRJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITStagingCommitMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITStagingCommitMRJob.java
new file mode 100644
index 0000000..76ad464
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITStagingCommitMRJob.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging.integration;
+
+import org.junit.Test;
+
+import org.hamcrest.core.StringContains;
+import org.hamcrest.core.StringEndsWith;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.AbstractITCommitMRJob;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingCommitter;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.apache.hadoop.fs.s3a.commit.staging.Paths.getMultipartUploadCommitsDirectory;
+
+/**
+ * Full integration test for the staging committer.
+ */
+public class ITStagingCommitMRJob extends AbstractITCommitMRJob {
+
+ @Override
+ protected String committerName() {
+ return StagingCommitter.NAME;
+ }
+
+ /**
+ * Verify that staging commit dirs are made absolute under the user's
+ * home directory, so, in a secure cluster, private.
+ */
+ @Test
+ public void testStagingDirectory() throws Throwable {
+ FileSystem hdfs = getDFS();
+ Configuration conf = hdfs.getConf();
+ conf.set(CommitConstants.FS_S3A_COMMITTER_STAGING_TMP_PATH,
+ "private");
+ Path dir = getMultipartUploadCommitsDirectory(conf, "UUID");
+ assertThat(dir.toString(), StringEndsWith.endsWith(
+ "UUID/"
+ + StagingCommitterConstants.STAGING_UPLOADS));
+ assertTrue("path unqualified", dir.isAbsolute());
+ String self = UserGroupInformation.getCurrentUser().getShortUserName();
+ assertThat(dir.toString(),
+ StringContains.containsString("/user/" + self + "/private"));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java
new file mode 100644
index 0000000..ea28ecb
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestDirectoryCommitProtocol.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging.integration;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjectionImpl;
+import org.apache.hadoop.fs.s3a.commit.staging.DirectoryStagingCommitter;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/** ITest of the low level protocol methods. */
+public class ITestDirectoryCommitProtocol extends ITestStagingCommitProtocol {
+
+ @Override
+ protected String suitename() {
+ return "ITestDirectoryCommitProtocol";
+ }
+
+ @Override
+ protected String getCommitterName() {
+ return CommitConstants.COMMITTER_NAME_DIRECTORY;
+ }
+
+ @Override
+ protected AbstractS3ACommitter createCommitter(
+ Path outputPath,
+ TaskAttemptContext context)
+ throws IOException {
+ return new DirectoryStagingCommitter(outputPath, context);
+ }
+
+ @Override
+ public AbstractS3ACommitter createFailingCommitter(
+ TaskAttemptContext tContext) throws IOException {
+ return new CommitterWithFailedThenSucceed(getOutDir(), tContext);
+ }
+
+ /**
+ * The class provides a overridden implementation of commitJobInternal which
+ * causes the commit failed for the first time then succeed.
+ */
+ private static final class CommitterWithFailedThenSucceed extends
+ DirectoryStagingCommitter implements CommitterFaultInjection {
+
+ private final CommitterFaultInjectionImpl injection;
+
+ CommitterWithFailedThenSucceed(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ injection = new CommitterFaultInjectionImpl(outputPath, context, true);
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ injection.setupJob(context);
+ super.setupJob(context);
+ }
+
+ @Override
+ public void abortJob(JobContext context, JobStatus.State state)
+ throws IOException {
+ injection.abortJob(context, state);
+ super.abortJob(context, state);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void cleanupJob(JobContext context) throws IOException {
+ injection.cleanupJob(context);
+ super.cleanupJob(context);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ injection.setupTask(context);
+ super.setupTask(context);
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ injection.commitTask(context);
+ super.commitTask(context);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ injection.abortTask(context);
+ super.abortTask(context);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ injection.commitJob(context);
+ super.commitJob(context);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ injection.needsTaskCommit(context);
+ return super.needsTaskCommit(context);
+ }
+
+ @Override
+ public void setFaults(CommitterFaultInjection.Faults... faults) {
+ injection.setFaults(faults);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestPartitionedCommitProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestPartitionedCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestPartitionedCommitProtocol.java
new file mode 100644
index 0000000..e3bc150
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestPartitionedCommitProtocol.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging.integration;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjectionImpl;
+import org.apache.hadoop.fs.s3a.commit.staging.DirectoryStagingCommitter;
+import org.apache.hadoop.fs.s3a.commit.staging.PartitionedStagingCommitter;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+
+/** ITest of the low level protocol methods. */
+public class ITestPartitionedCommitProtocol extends ITestStagingCommitProtocol {
+
+ @Override
+ protected String suitename() {
+ return "ITestPartitionedCommitProtocol";
+ }
+
+ @Override
+ protected String getCommitterName() {
+ return CommitConstants.COMMITTER_NAME_PARTITIONED;
+ }
+
+ @Override
+ protected AbstractS3ACommitter createCommitter(
+ Path outputPath,
+ TaskAttemptContext context)
+ throws IOException {
+ return new PartitionedStagingCommitter(outputPath, context);
+ }
+
+ @Override
+ public AbstractS3ACommitter createFailingCommitter(
+ TaskAttemptContext tContext) throws IOException {
+ return new CommitterWithFailedThenSucceed(getOutDir(), tContext);
+ }
+
+ @Override
+ public void testMapFileOutputCommitter() throws Exception {
+ skip("Partioning committer is not suitable for Map Output");
+ }
+
+ /**
+ * The class provides a overridden implementation of commitJobInternal which
+ * causes the commit failed for the first time then succeed.
+ */
+ private static final class CommitterWithFailedThenSucceed extends
+ DirectoryStagingCommitter implements CommitterFaultInjection {
+ private final CommitterFaultInjectionImpl injection;
+
+ CommitterWithFailedThenSucceed(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ injection = new CommitterFaultInjectionImpl(outputPath, context, true);
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ injection.setupJob(context);
+ super.setupJob(context);
+ }
+
+ @Override
+ public void abortJob(JobContext context, JobStatus.State state)
+ throws IOException {
+ injection.abortJob(context, state);
+ super.abortJob(context, state);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void cleanupJob(JobContext context) throws IOException {
+ injection.cleanupJob(context);
+ super.cleanupJob(context);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ injection.setupTask(context);
+ super.setupTask(context);
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ injection.commitTask(context);
+ super.commitTask(context);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ injection.abortTask(context);
+ super.abortTask(context);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ injection.commitJob(context);
+ super.commitJob(context);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ injection.needsTaskCommit(context);
+ return super.needsTaskCommit(context);
+ }
+
+ @Override
+ public void setFaults(CommitterFaultInjection.Faults... faults) {
+ injection.setFaults(faults);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java
new file mode 100644
index 0000000..08c572e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/integration/ITestStagingCommitProtocol.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging.integration;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.InconsistentS3ClientFactory;
+import org.apache.hadoop.fs.s3a.S3ClientFactory;
+import org.apache.hadoop.fs.s3a.commit.AbstractITCommitProtocol;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjectionImpl;
+import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.fs.s3a.commit.staging.Paths;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingCommitter;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import static org.apache.hadoop.fs.s3a.Constants.S3_CLIENT_FACTORY_IMPL;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/** Test the staging committer's handling of the base protocol operations. */
+public class ITestStagingCommitProtocol extends AbstractITCommitProtocol {
+
+ @Override
+ protected String suitename() {
+ return "ITestStagingCommitProtocol";
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setInt(FS_S3A_COMMITTER_THREADS, 1);
+ // switch to the inconsistent filesystem
+ conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
+ S3ClientFactory.class);
+
+ // disable unique filenames so that the protocol tests of FileOutputFormat
+ // and this test generate consistent names.
+ conf.setBoolean(FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES, false);
+ return conf;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ // identify working dir for staging and delete
+ Configuration conf = getConfiguration();
+ String uuid = StagingCommitter.getUploadUUID(conf,
+ getTaskAttempt0().getJobID());
+ Path tempDir = Paths.getLocalTaskAttemptTempDir(conf, uuid,
+ getTaskAttempt0());
+ rmdir(tempDir, conf);
+ }
+
+ @Override
+ protected String getCommitterName() {
+ return InternalCommitterConstants.COMMITTER_NAME_STAGING;
+ }
+
+ @Override
+ protected AbstractS3ACommitter createCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new StagingCommitter(outputPath, context);
+ }
+
+ public AbstractS3ACommitter createFailingCommitter(
+ TaskAttemptContext tContext) throws IOException {
+ return new CommitterWithFailedThenSucceed(getOutDir(), tContext);
+ }
+
+ @Override
+ protected boolean shouldExpectSuccessMarker() {
+ return false;
+ }
+
+ @Override
+ protected void expectJobCommitToFail(JobContext jContext,
+ AbstractS3ACommitter committer) throws Exception {
+ expectJobCommitFailure(jContext, committer,
+ IOException.class);
+ }
+
+ protected void validateTaskAttemptPathDuringWrite(Path p) throws IOException {
+ // this is expected to be local FS
+ ContractTestUtils.assertPathExists(getLocalFS(), "task attempt", p);
+ }
+
+ protected void validateTaskAttemptPathAfterWrite(Path p) throws IOException {
+ // this is expected to be local FS
+ ContractTestUtils.assertPathExists(getLocalFS(), "task attempt", p);
+ }
+
+ protected FileSystem getLocalFS() throws IOException {
+ return FileSystem.getLocal(getConfiguration());
+ }
+
+ /**
+ * The class provides a overridden implementation of commitJobInternal which
+ * causes the commit failed for the first time then succeed.
+ */
+ private static final class CommitterWithFailedThenSucceed extends
+ StagingCommitter implements CommitterFaultInjection {
+
+ private final CommitterFaultInjectionImpl injection;
+
+ CommitterWithFailedThenSucceed(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ injection = new CommitterFaultInjectionImpl(outputPath, context, true);
+ }
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ injection.setupJob(context);
+ super.setupJob(context);
+ }
+
+ @Override
+ public void abortJob(JobContext context, JobStatus.State state)
+ throws IOException {
+ injection.abortJob(context, state);
+ super.abortJob(context, state);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void cleanupJob(JobContext context) throws IOException {
+ injection.cleanupJob(context);
+ super.cleanupJob(context);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ injection.setupTask(context);
+ super.setupTask(context);
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ injection.commitTask(context);
+ super.commitTask(context);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ injection.abortTask(context);
+ super.abortTask(context);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ injection.commitJob(context);
+ super.commitJob(context);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ injection.needsTaskCommit(context);
+ return super.needsTaskCommit(context);
+ }
+
+ @Override
+ public void setFaults(Faults... faults) {
+ injection.setFaults(faults);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
index a33c001..22a028a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/AbstractS3GuardToolTestBase.java
@@ -35,11 +35,14 @@ import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.StringUtils;
+import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.E_BAD_STATE;
import static org.apache.hadoop.fs.s3a.s3guard.S3GuardTool.SUCCESS;
import static org.apache.hadoop.test.LambdaTestUtils.intercept;
@@ -235,6 +238,26 @@ public abstract class AbstractS3GuardToolTestBase extends AbstractS3ATestBase {
});
}
+ @Test
+ public void testProbeForMagic() throws Throwable {
+ S3AFileSystem fs = getFileSystem();
+ String name = fs.getUri().toString();
+ S3GuardTool.BucketInfo cmd = new S3GuardTool.BucketInfo(
+ getConfiguration());
+ if (fs.hasCapability(
+ CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER)) {
+ // if the FS is magic, expect this to work
+ exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name);
+ } else {
+ // if the FS isn't magic, expect the probe to fail
+ ExitUtil.ExitException e = intercept(ExitUtil.ExitException.class,
+ () -> exec(cmd, S3GuardTool.BucketInfo.MAGIC_FLAG, name));
+ if (e.getExitCode() != E_BAD_STATE) {
+ throw e;
+ }
+ }
+ }
+
/**
* Get the test CSV file; assume() that it is not modified (i.e. we haven't
* switched to a new storage infrastructure where the bucket is no longer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
index 02eb7b8..5763b83 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/s3guard/TestDynamoDBMetadataStore.java
@@ -537,7 +537,7 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
}
@Test
- public void testDeleteTable() throws IOException {
+ public void testDeleteTable() throws Exception {
final String tableName = "testDeleteTable";
final S3AFileSystem s3afs = getFileSystem();
final Configuration conf = s3afs.getConf();
@@ -553,7 +553,6 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
// delete table once more; be ResourceNotFoundException swallowed silently
ddbms.destroy();
verifyTableNotExist(tableName);
-
try {
// we can no longer list the destroyed table
ddbms.listChildren(new Path(S3URI));
@@ -582,13 +581,9 @@ public class TestDynamoDBMetadataStore extends MetadataStoreTestBase {
*
* This should not rely on the {@link DynamoDBMetadataStore} implementation.
*/
- private static void verifyTableNotExist(String tableName) {
- final Table table = dynamoDB.getTable(tableName);
- try {
- table.describe();
- fail("Expecting ResourceNotFoundException for table '" + tableName + "'");
- } catch (ResourceNotFoundException ignored) {
- }
+ private static void verifyTableNotExist(String tableName) throws Exception{
+ intercept(ResourceNotFoundException.class,
+ () -> dynamoDB.getTable(tableName).describe());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md
new file mode 100644
index 0000000..c6dbf55
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/committers.md
@@ -0,0 +1,819 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Committing work to S3 with the "S3A Committers"
+
+<!-- MACRO{toc|fromDepth=0|toDepth=5} -->
+
+This page covers the S3A Committers, which can commit work directly
+to an S3 object store.
+
+These committers are designed to solve a fundamental problem which
+the standard committers of work cannot do to S3: consistent, high performance,
+and reliable commitment of output to S3.
+
+For details on their internal design, see
+[S3A Committers: Architecture and Implementation](./committer_architecture.html).
+
+
+## Introduction: The Commit Problem
+
+
+Apache Hadoop MapReduce (and behind the scenes, Apache Spark) often write
+the output of their work to filesystems
+
+Normally, Hadoop uses the `FileOutputFormatCommitter` to manage the
+promotion of files created in a single task attempt to the final output of
+a query. This is done in a way to handle failures of tasks and jobs, and to
+support speculative execution. It does that by listing directories and renaming
+their content into the final destination when tasks and then jobs are committed.
+
+This has some key requirement of the underlying filesystem:
+
+1. When you list a directory, you see all the files which have been created in it,
+and no files which are not in it (i.e. have been deleted).
+1. When you rename a directory, it is an `O(1)` atomic transaction. No other
+process across the cluster may rename a file or directory to the same path.
+If the rename fails for any reason, either the data is at the original location,
+or it is at the destination, -in which case the rename actually succeeded.
+
+**The S3 object store and the `s3a://` filesystem client cannot meet these requirements.*
+
+1. Amazon S3 has inconsistent directory listings unless S3Guard is enabled.
+1. The S3A mimics `rename()` by copying files and then deleting the originals.
+This can fail partway through, and there is nothing to prevent any other process
+in the cluster attempting a rename at the same time.
+
+As a result,
+
+* Files my not be listed, hence not renamed into place.
+* Deleted files may still be discovered, confusing the rename process to the point
+of failure.
+* If a rename fails, the data is left in an unknown state.
+* If more than one process attempts to commit work simultaneously, the output
+directory may contain the results of both processes: it is no longer an exclusive
+operation.
+*. While S3Guard may deliver the listing consistency, commit time is still
+proportional to the amount of data created. It still can't handle task failure.
+
+**Using the "classic" `FileOutputCommmitter` to commit work to Amazon S3 risks
+loss or corruption of generated data**
+
+
+To address these problems there is now explicit support in the `hadop-aws`
+module for committing work to Amazon S3 via the S3A filesystem client,
+*the S3A Committers*
+
+
+For safe, as well as high-performance output of work to S3,
+we need use "a committer" explicitly written to work with S3, treating it as
+an object store with special features.
+
+
+### Background : Hadoop's "Commit Protocol"
+
+How exactly is work written to its final destination? That is accomplished by
+a "commit protocol" between the workers and the job manager.
+
+This protocol is implemented in Hadoop MapReduce, with a similar but extended
+version in Apache Spark:
+
+1. A "Job" is the entire query, with inputs to outputs
+1. The "Job Manager" is the process in charge of choreographing the execution
+of the job. It may perform some of the actual computation too.
+1. The job has "workers", which are processes which work the actual data
+and write the results.
+1. Workers execute "Tasks", which are fractions of the job, a job whose
+input has been *partitioned* into units of work which can be executed independently.
+1. The Job Manager directs workers to execute "tasks", usually trying to schedule
+the work close to the data (if the filesystem provides locality information).
+1. Workers can fail: the Job manager needs to detect this and reschedule their active tasks.
+1. Workers can also become separated from the Job Manager, a "network partition".
+It is (provably) impossible for the Job Manager to distinguish a running-but-unreachable
+worker from a failed one.
+1. The output of a failed task must not be visible; this is to avoid its
+data getting into the final output.
+1. Multiple workers can be instructed to evaluate the same partition of the work;
+this "speculation" delivers speedup as it can address the "straggler problem".
+When multiple workers are working on the same data, only one worker is allowed
+to write the final output.
+1. The entire job may fail (often from the failure of the Job Manager (MR Master, Spark Driver, ...)).
+1, The network may partition, with workers isolated from each other or
+the process managing the entire commit.
+1. Restarted jobs may recover from a failure by reusing the output of all
+completed tasks (MapReduce with the "v1" algorithm), or just by rerunning everything
+(The "v2" algorithm and Spark).
+
+
+What is "the commit protocol" then? It is the requirements on workers as to
+when their data is made visible, where, for a filesystem, "visible" means "can
+be seen in the destination directory of the query."
+
+* There is a destination directory of work, "the output directory."
+* The final output of tasks must be in this directory *or paths underneath it*.
+* The intermediate output of a task must not be visible in the destination directory.
+That is: they must not write directly to the destination.
+* The final output of a task *may* be visible under the destination.
+* The Job Manager makes the decision as to whether a task's data is to be "committed",
+be it directly to the final directory or to some intermediate store..
+* Individual workers communicate with the Job manager to manage the commit process:
+whether the output is to be *committed* or *aborted*
+* When a worker commits the output of a task, it somehow promotes its intermediate work to becoming
+final.
+* When a worker aborts a task's output, that output must not become visible
+(i.e. it is not committed).
+* Jobs themselves may be committed/aborted (the nature of "when" is not covered here).
+* After a Job is committed, all its work must be visible.
+* And a file `_SUCCESS` may be written to the output directory.
+* After a Job is aborted, all its intermediate data is lost.
+* Jobs may also fail. When restarted, the successor job must be able to clean up
+all the intermediate and committed work of its predecessor(s).
+* Task and Job processes measure the intervals between communications with their
+Application Master and YARN respectively.
+When the interval has grown too large they must conclude
+that the network has partitioned and that they must abort their work.
+
+
+That's "essentially" it. When working with HDFS and similar filesystems,
+directory `rename()` is the mechanism used to commit the work of tasks and
+jobs.
+* Tasks write data to task attempt directories under the directory `_temporary`
+underneath the final destination directory.
+* When a task is committed, these files are renamed to the destination directory
+(v2 algorithm) or a job attempt directory under `_temporary` (v1 algorithm).
+* When a job is committed, for the v2 algorithm the `_SUCCESS` file is created,
+and the `_temporary` deleted.
+* For the v1 algorithm, when a job is committed, all the tasks committed under
+the job attempt directory will have their output renamed into the destination
+directory.
+* The v2 algorithm recovers from failure by deleting the destination directory
+and restarting the job.
+* The v1 algorithm recovers from failure by discovering all committed tasks
+whose output is in the job attempt directory, *and only rerunning all uncommitted tasks*.
+
+
+None of this algorithm works safely or swiftly when working with "raw" AWS S3 storage:
+* Directory listing can be inconsistent: the tasks and jobs may not list all work to
+be committed.
+* Renames go from being fast, atomic operations to slow operations which can fail partway through.
+
+This then is the problem which the S3A committers address:
+
+*How to safely and reliably commit work to Amazon S3 or compatible object store*
+
+
+## Meet the S3A Commmitters
+
+Since Hadoop 3.1, the S3A FileSystem has been accompanied by classes
+designed to integrate with the Hadoop and Spark job commit protocols, classes
+which interact with the S3A filesystem to reliably commit work work to S3:
+*The S3A Committers*
+
+The underlying architecture of this process is very complex, and
+covered in [the committer architecture documentation](./committer_architecture.html).
+
+The key concept to know of is S3's "Multipart Upload" mechanism. This allows
+an S3 client to write data to S3 in multiple HTTP POST requests, only completing
+the write operation with a final POST to complete the upload; this final POST
+consisting of a short list of the etags of the uploaded blocks.
+This multipart upload mechanism is already automatically used when writing large
+amounts of data to S3; an implementation detail of the S3A output stream.
+
+The S3A committers make explicit use of this multipart upload ("MPU") mechanism:
+
+1. The individual *tasks* in a job write their data to S3 as POST operations
+within multipart uploads, yet do not issue the final POST to complete the upload.
+1. The multipart uploads are committed in the job commit process.
+
+There are two different S3A committer types, *staging*
+and *magic*. The committers primarily vary in how data is written during task execution,
+how the pending commit information is passed to the job manager, and in how
+conflict with existing files is resolved.
+
+
+| feature | staging | magic |
+|--------|---------|---|
+| task output destination | local disk | S3A *without completing the write* |
+| task commit process | upload data from disk to S3 | list all pending uploads on s3 and write details to job attempt directory |
+| task abort process | delete local disk data | list all pending uploads and abort them |
+| job commit | list & complete pending uploads | list & complete pending uploads |
+
+The other metric is "maturity". There, the fact that the Staging committers
+are based on Netflix's production code counts in its favor.
+
+
+### The Staging Committer
+
+This is based on work from Netflix. It "stages" data into the local filesystem.
+It also requires the cluster to have HDFS, so that
+
+Tasks write to URLs with `file://` schemas. When a task is committed,
+its files are listed, uploaded to S3 as incompleted Multipart Uploads.
+The information needed to complete the uploads is saved to HDFS where
+it is committed through the standard "v1" commit algorithm.
+
+When the Job is committed, the Job Manager reads the lists of pending writes from its
+HDFS Job destination directory and completes those uploads.
+
+Cancelling a task is straighforward: the local directory is deleted with
+its staged data. Cancelling a job is achieved by reading in the lists of
+pending writes from the HDFS job attempt directory, and aborting those
+uploads. For extra safety, all outstanding multipart writes to the destination directory
+are aborted.
+
+The staging committer comes in two slightly different forms, with slightly
+diffrent conflict resolution policies:
+
+
+* **Directory**: the entire directory tree of data is written or overwritten,
+as normal.
+
+* **Partitioned**: special handling of partitioned directory trees of the form
+`YEAR=2017/MONTH=09/DAY=19`: conflict resolution is limited to the partitions
+being updated.
+
+
+The Partitioned Committer is intended to allow jobs updating a partitioned
+directory tree to restrict the conflict resolution to only those partition
+directories containing new data. It is intended for use with Apache Spark
+only.
+
+
+## Conflict Resolution in the Staging Committers
+
+The Staging committers offer the ability to replace the conflict policy
+of the execution engine with policy designed to work with the tree of data.
+This is based on the experience and needs of Netflix, where efficiently adding
+new data to an existing partitioned directory tree is a common operation.
+
+```xml
+<property>
+ <name>fs.s3a.committer.staging.conflict-mode</name>
+ <value>fail</value>
+ <description>
+ Staging committer conflict resolution policy: {@value}.
+ Supported: fail, append, replace.
+ </description>
+</property>
+```
+
+**replace** : when the job is committed (and not before), delete files in
+directories into which new data will be written.
+
+**fail**: when there are existing files in the destination, fail the job.
+
+**append**: Add new data to the directories at the destination; overwriting
+any with the same name. Reliable use requires unique names for generated files,
+which the committers generate
+by default.
+
+The difference between the two staging ommitters are as follows:
+
+The Directory Committer uses the entire directory tree for conflict resolution.
+If any file exists at the destination it will fail in job setup; if the resolution
+mechanism is "replace" then all existing files will be deleted.
+
+The partitioned committer calculates the partitions into which files are added,
+the final directories in the tree, and uses that in its conflict resolution
+process:
+
+
+**replace** : delete all data in the destination partition before committing
+the new files.
+
+**fail**: fail if there is data in the destination partition, ignoring the state
+of any parallel partitions.
+
+**append**: add the new data.
+
+It's intended for use in Apache Spark Dataset operations, rather
+than Hadoop's original MapReduce engine, and only in jobs
+where adding new data to an existing dataset is the desired goal.
+
+Preequisites for successful work
+
+1. The output is written into partitions via `PARTITIONED BY` or `partitionedBy()`
+instructions.
+2. There is no data written directly to the root path (all files there are
+ignored; it's implicitly "append").
+
+Here's an example in Spark, assuming that `sourceDataset` is a dataset
+whose columns include "year" and "month":
+
+```scala
+sourceDataset
+ .write
+ .partitionBy("year", "month")
+ .mode(SaveMode.Append)
+ .opt("fs.s3a.committer.name", "partitioned")
+ .opt("fs.s3a.committer.staging.conflict-mode", "replace")
+ .format("orc")
+ .save("s3a://examples/statistics")
+```
+
+
+### The Magic Committer
+
+The "Magic" committer does its work through "magic" in the filesystem:
+attempts to write to specific "magic" paths are interpreted as writes
+to a parent directory *which are not to be completed*. When the output stream
+is closed, the information needed to complete the write is saved in the magic
+directory. The task committer saves the list of these to a directory for the
+job committers use, or, if aborting, lists the pending writes and aborts them.
+
+The job committer reads in the list of pending commits, and commits them as
+the Staging Committer does.
+
+Compared to the Staging Committer, the Magic Committer offers faster write
+times: output is uploaded to S3 as it is written, rather than in the
+task commit.
+
+However, it has extra requirements of the filesystem
+
+1. It requires a consistent object store, which for Amazon S3,
+means that [S3Guard](./s3guard.html) must be enabled. For third-party stores,
+consult the documentation.
+1. The S3A client must be configured to recognize interactions
+with the magic directories and treat them specially.
+
+
+It's also not been field tested to the extent of Netflix's committer; consider
+it the least mature of the committers.
+
+
+#### Which Committer to Use?
+
+1. If you want to create or update existing partitioned data trees in Spark, use thee
+Partitioned Committer. Make sure you have enough hard disk capacity for all staged data.
+Do not use it in other situations.
+
+1. If you know that your object store is consistent, or that the processes
+writing data use S3Guard, use the Magic Committer for higher performance
+writing of large amounts of data.
+
+1. Otherwise: use the directory committer, making sure you have enough
+hard disk capacity for all staged data.
+
+Put differently: start with the Directory Committer.
+
+## Switching to an S3A Committer
+
+To use an S3A committer, the property `mapreduce.outputcommitter.factory.scheme.s3a`
+must be set to the S3A committer factory, `org.apache.hadoop.fs.s3a.commit.staging.S3ACommitterFactory`.
+This is done in `core-default.xml`
+
+```xml
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.s3a</name>
+ <value>org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory</value>
+ <description>
+ The committer factory to use when writing data to S3A filesystems.
+ </description>
+</property>
+```
+
+What is missing is an explicit choice of committer to use in the property
+`fs.s3a.committer.name`; so the classic (and unsafe) file committer is used.
+
+| `fs.s3a.committer.name` | Committer |
+|--------|---------|
+| `directory` | directory staging committer |
+| `partitioned` | partition staging committer (for use in Spark only) |
+| `magic` | the "magic" committer |
+| `file` | the original and unsafe File committer; (default) |
+
+
+
+## Using the Directory and Partitioned Staging Committers
+
+Generated files are initially written to a local directory underneath one of the temporary
+directories listed in `fs.s3a.buffer.dir`.
+
+
+The staging commmitter needs a path in the cluster filesystem
+(e.g. HDFS). This must be declared in `fs.s3a.committer.tmp.path`.
+
+Temporary files are saved in HDFS (or other cluster filesystem) under the path
+`${fs.s3a.committer.tmp.path}/${user}` where `user` is the name of the user running the job.
+The default value of `fs.s3a.committer.tmp.path` is `tmp/staging`,
+Which will be converted at run time to a path under the current user's home directory,
+essentially `~/tmp/staging`
+ so the temporary directory
+
+The application attempt ID is used to create a unique path under this directory,
+resulting in a path `~/tmp/staging/${user}/${application-attempt-id}/` under which
+summary data of each task's pending commits are managed using the standard
+`FileOutputFormat` committer.
+
+When a task is committed the data is uploaded under the destination directory.
+The policy of how to react if the destination exists is defined by
+the `fs.s3a.committer.staging.conflict-mode` setting.
+
+| `fs.s3a.committer.staging.conflict-mode` | Meaning |
+| -----------------------------------------|---------|
+| `fail` | Fail if the destination directory exists |
+| `replace` | Delete all existing files before committing the new data |
+| `append` | Add the new files to the existing directory tree |
+
+
+## The "Partitioned" Staging Committer
+
+This committer an extension of the "Directory" committer which has a special conflict resolution
+policy designed to support operations which insert new data into a directory tree structured
+using Hive's partitioning strategy: different levels of the tree represent different columns.
+
+For example, log data could be partitioned by `YEAR` and then by `MONTH`, with different
+entries underneath.
+
+```
+logs/YEAR=2017/MONTH=01/
+ log-20170101.avro
+ log-20170102.avro
+ ...
+ log-20170131.avro
+
+logs/YEAR=2017/MONTH=02/
+ log-20170201.avro
+ log-20170202.avro
+ ...
+ log-20170227.avro
+
+logs/YEAR=2017/MONTH=03/
+logs/YEAR=2017/MONTH=04/
+```
+
+A partitioned structure like this allows for queries using Hive or Spark to filter out
+files which do not contain relevant data.
+
+What the partitioned committer does is, where the tooling permits, allows callers
+to add data to an existing partitioned layout*.
+
+More specifically, it does this by having a conflict resolution options which
+only act on invididual partitions, rather than across the entire output tree.
+
+| `fs.s3a.committer.staging.conflict-mode` | Meaning |
+| -----------------------------------------|---------|
+| `fail` | Fail if the destination partition(s) exist |
+| `replace` | Delete the existing data partitions before committing the new data |
+| `append` | Add the new data to the existing partitions |
+
+
+As an example, if a job was writing the file
+`logs/YEAR=2017/MONTH=02/log-20170228.avro`, then with a policy of `fail`,
+the job would fail. With a policy of `replace`, then entire directory
+`logs/YEAR=2017/MONTH=02/` would be deleted before the new file `log-20170228.avro`
+was written. With the policy of `append`, the new file would be added to
+the existing set of files.
+
+
+### Notes
+
+1. A deep partition tree can itself be a performance problem in S3 and the s3a client,
+or, more specifically. a problem with applications which use recursive directory tree
+walks to work with data.
+
+1. The outcome if you have more than one job trying simultaneously to write data
+to the same destination with any policy other than "append" is undefined.
+
+1. In the `append` operation, there is no check for conflict with file names.
+If, in the example above, the file `log-20170228.avro` already existed,
+it would be overridden. Set `fs.s3a.committer.staging.unique-filenames` to `true`
+to ensure that a UUID is included in every filename to avoid this.
+
+
+## Using the Magic committer
+
+This is less mature than the Staging Committer, but promises higher
+performance.
+
+### FileSystem client setup
+
+1. Use a *consistent* S3 object store. For Amazon S3, this means enabling
+[S3Guard](./s3guard.html). For S3-compatible filesystems, consult the filesystem
+documentation to see if it is consistent, hence compatible "out of the box".
+1. Turn the magic on by `fs.s3a.committer.magic.enabled"`
+
+```xml
+<property>
+ <name>fs.s3a.committer.magic.enabled</name>
+ <description>
+ Enable support in the filesystem for the S3 "Magic" committter.
+ </description>
+ <value>true</value>
+</property>
+```
+
+*Do not use the Magic Committer on an inconsistent S3 object store. For
+Amazon S3, that means S3Guard must *always* be enabled.
+
+
+### Enabling the committer
+
+```xml
+<property>
+ <name>fs.s3a.committer.name</name>
+ <value>magic</value>
+</property>
+
+```
+
+Conflict management is left to the execution engine itself.
+
+## Committer Configuration Options
+
+
+| Option | Magic | Directory | Partitioned | Meaning | Default |
+|--------|-------|-----------|-------------|---------|---------|
+| `mapreduce.fileoutputcommitter.marksuccessfuljobs` | X | X | X | Write a `_SUCCESS` file at the end of each job | `true` |
+| `fs.s3a.committer.threads` | X | X | X | Number of threads in committers for parallel operations on files. | 8 |
+| `fs.s3a.committer.staging.conflict-mode` | | X | X | Conflict resolution: `fail`, `abort` or `overwrite`| `fail` |
+| `fs.s3a.committer.staging.unique-filenames` | | X | X | Generate unique filenames | `true` |
+
+| `fs.s3a.committer.magic.enabled` | X | | | Enable "magic committer" support in the filesystem | `false` |
+
+
+
+
+| Option | Magic | Directory | Partitioned | Meaning | Default |
+|--------|-------|-----------|-------------|---------|---------|
+| `fs.s3a.buffer.dir` | X | X | X | Local filesystem directory for data being written and/or staged. | |
+| `fs.s3a.committer.staging.tmp.path` | | X | X | Path in the cluster filesystem for temporary data | `tmp/staging` |
+
+
+```xml
+<property>
+ <name>fs.s3a.committer.name</name>
+ <value>file</value>
+ <description>
+ Committer to create for output to S3A, one of:
+ "file", "directory", "partitioned", "magic".
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.magic.enabled</name>
+ <value>false</value>
+ <description>
+ Enable support in the filesystem for the S3 "Magic" committer.
+ When working with AWS S3, S3Guard must be enabled for the destination
+ bucket, as consistent metadata listings are required.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.threads</name>
+ <value>8</value>
+ <description>
+ Number of threads in committers for parallel operations on files
+ (upload, commit, abort, delete...)
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.tmp.path</name>
+ <value>tmp/staging</value>
+ <description>
+ Path in the cluster filesystem for temporary data.
+ This is for HDFS, not the local filesystem.
+ It is only for the summary data of each file, not the actual
+ data being committed.
+ Using an unqualified path guarantees that the full path will be
+ generated relative to the home directory of the user creating the job,
+ hence private (assuming home directory permissions are secure).
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.unique-filenames</name>
+ <value>true</value>
+ <description>
+ Option for final files to have a unique name through job attempt info,
+ or the value of fs.s3a.committer.staging.uuid
+ When writing data with the "append" conflict option, this guarantees
+ that new data will not overwrite any existing data.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.committer.staging.conflict-mode</name>
+ <value>fail</value>
+ <description>
+ Staging committer conflict resolution policy.
+ Supported: "fail", "append", "replace".
+ </description>
+</property>
+
+<property>
+ <name>s.s3a.committer.staging.abort.pending.uploads</name>
+ <value>true</value>
+ <description>
+ Should the staging committers abort all pending uploads to the destination
+ directory?
+
+ Changing this if more than one partitioned committer is
+ writing to the same destination tree simultaneously; otherwise
+ the first job to complete will cancel all outstanding uploads from the
+ others. However, it may lead to leaked outstanding uploads from failed
+ tasks. If disabled, configure the bucket lifecycle to remove uploads
+ after a time period, and/or set up a workflow to explicitly delete
+ entries. Otherwise there is a risk that uncommitted uploads may run up
+ bills.
+ </description>
+</property>
+
+<property>
+ <name>mapreduce.outputcommitter.factory.scheme.s3a</name>
+ <value>org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory</value>
+ <description>
+ The committer factory to use when writing data to S3A filesystems.
+ If mapreduce.outputcommitter.factory.class is set, it will
+ override this property.
+
+ (This property is set in mapred-default.xml)
+ </description>
+</property>
+
+```
+
+
+## Troubleshooting
+
+### `Filesystem does not have support for 'magic' committer`
+
+```
+org.apache.hadoop.fs.s3a.commit.PathCommitException: `s3a://landsat-pds': Filesystem does not have support for 'magic' committer enabled
+in configuration option fs.s3a.committer.magic.enabled
+```
+
+The Job is configured to use the magic committer, but the S3A bucket has not been explicitly
+declared as supporting it.
+
+The destination bucket **must** be declared as supporting the magic committer.
+
+This can be done for those buckets which are known to be consistent, either
+because [S3Guard](s3guard.html) is used to provide consistency,
+or because the S3-compatible filesystem is known to be strongly consistent.
+
+```xml
+<property>
+ <name>fs.s3a.bucket.landsat-pds.committer.magic.enabled</name>
+ <value>true</value>
+</property>
+```
+
+*IMPORTANT*: only enable the magic committer against object stores which
+offer consistent listings. By default, Amazon S3 does not do this -which is
+why the option `fs.s3a.committer.magic.enabled` is disabled by default.
+
+
+Tip: you can verify that a bucket supports the magic committer through the
+`hadoop s3guard bucket-info` command:
+
+
+```
+> hadoop s3guard bucket-info -magic s3a://landsat-pds/
+
+Filesystem s3a://landsat-pds
+Location: us-west-2
+Filesystem s3a://landsat-pds is not using S3Guard
+The "magic" committer is not supported
+
+S3A Client
+ Endpoint: fs.s3a.endpoint=(unset)
+ Encryption: fs.s3a.server-side-encryption-algorithm=none
+ Input seek policy: fs.s3a.experimental.input.fadvise=normal
+2017-09-27 19:18:57,917 INFO util.ExitUtil: Exiting with status 46: 46: The magic committer is not enabled for s3a://landsat-pds
+```
+
+## Error message: "File being created has a magic path, but the filesystem has magic file support disabled:
+
+A file is being written to a path which is used for "magic" files,
+files which are actually written to a different destination than their stated path
+*but the filesystem doesn't support "magic" files*
+
+This message should not appear through the committer itself —it will
+fail with the error message in the previous section, but may arise
+if other applications are attempting to create files under the path `/__magic/`.
+
+Make sure the filesytem meets the requirements of the magic committer
+(a consistent S3A filesystem through S3Guard or the S3 service itself),
+and set the `fs.s3a.committer.magic.enabled` flag to indicate that magic file
+writes are supported.
+
+
+### `FileOutputCommitter` appears to be still used (from logs or delays in commits)
+
+The Staging committers use the original `FileOutputCommitter` to manage
+the propagation of commit information: do not worry if it the logs show `FileOutputCommitter`
+work with data in the cluster filesystem (e.g. HDFS).
+
+One way to make sure that the `FileOutputCommitter` is not being used to write
+the data to S3 is to set the option `mapreduce.fileoutputcommitter.algorithm.version`
+to a value such as "10". Because the only supported algorithms are "1" and "2",
+any erroneously created `FileOutputCommitter` will raise an exception in its constructor
+when instantiated:
+
+```
+java.io.IOException: Only 1 or 2 algorithm version is supported
+at org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.<init>(FileOutputCommitter.java:130)
+at org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter.<init>(FileOutputCommitter.java:104)
+at org.apache.parquet.hadoop.ParquetOutputCommitter.<init>(ParquetOutputCommitter.java:42)
+at org.apache.parquet.hadoop.ParquetOutputFormat.getOutputCommitter(ParquetOutputFormat.java:395)
+at org.apache.spark.internal.io.HadoopMapReduceCommitProtocol.setupCommitter(HadoopMapReduceCommitProtocol.scala:67)
+at com.hortonworks.spark.cloud.commit.PathOutputCommitProtocol.setupCommitter(PathOutputCommitProtocol.scala:62)
+at org.apache.spark.internal.io.HadoopMapReduceCommitProtocol.setupJob(HadoopMapReduceCommitProtocol.scala:124)
+at com.hortonworks.spark.cloud.commit.PathOutputCommitProtocol.setupJob(PathOutputCommitProtocol.scala:152)
+at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:175)
+at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:145)
+```
+
+While that will not make the problem go away, it will at least make
+the failure happen at the start of a job.
+
+(Setting this option will not interfer with the Staging Committers' use of HDFS,
+as it explicitly sets the algorithm to "2" for that part of its work).
+
+The other way to check which committer to use is to examine the `_SUCCESS` file.
+If it is 0-bytes long, the classic `FileOutputCommitter` committed the job.
+The S3A committers all write a non-empty JSON file; the `committer` field lists
+the committer used.
+
+
+*Common causes*
+
+1. The property `fs.s3a.committer.name` is set to "file". Fix: change.
+1. The job has overridden the property `mapreduce.outputcommitter.factory.class`
+with a new factory class for all committers. This takes priority over
+all committers registered for the s3a:// schema.
+1. The property `mapreduce.outputcommitter.factory.scheme.s3a` is unset.
+1. The output format has overridden `FileOutputFormat.getOutputCommitter()`
+and is returning its own committer -one which is a subclass of `FileOutputCommitter`.
+
+That final cause. *the output format is returning its own committer*, is not
+easily fixed; it may be that the custom committer performs critical work
+during its lifecycle, and contains assumptions about the state of the written
+data during task and job commit (i.e. it is in the destination filesystem).
+Consult with the authors/maintainers of the output format
+to see whether it would be possible to integrate with the new committer factory
+mechanism and object-store-specific commit algorithms.
+
+Parquet is a special case here: its committer does no extra work
+other than add the option to read all newly-created files then write a schema
+summary. The Spark integration has explicit handling for Parquet to enable it
+to support the new committers, removing this (slow on S3) option.
+
+If you have subclassed `FileOutputCommitter` and want to move to the
+factory model, please get in touch.
+
+
+## Job/Task fails with PathExistsException: Destination path exists and committer conflict resolution mode is "fail"
+
+This surfaces when either of two conditions are met.
+
+1. The Directory committer is used with `fs.s3a.committer.staging.conflict-mode` set to
+`fail` and the output/destination directory exists.
+The job will fail in the driver during job setup.
+1. The Partitioned Committer is used with `fs.s3a.committer.staging.conflict-mode` set to
+`fail` and one of the partitions. The specific task(s) generating conflicting data will fail
+during task commit, which will cause the entire job to fail.
+
+If you are trying to write data and want write conflicts to be rejected, this is the correct
+behavior: there was data at the destination so the job was aborted.
+
+## Staging committer task fails with IOException: No space left on device
+
+There's not enough space on the local hard disk (real or virtual)
+to store all the uncommitted data of the active tasks on that host.
+Because the staging committers write all output to the local disk
+and only upload the data on task commits, enough local temporary
+storage is needed to store all output generated by all uncommitted
+tasks running on the single host. Small EC2 VMs may run out of disk.
+
+1. Make sure that `fs.s3a.buffer.dir` includes a temporary directory on
+every available hard disk; this spreads load better.
+
+1. Add more disk space. In EC2: request instances with more local storage.
+There is no need for EMR storage; this is just for temporary data.
+
+1. Purge the directories listed in `fs.s3a.buffer.dir` of old data.
+Failed tasks may not clean up all old files.
+
+1. Reduce the number of worker threads/process in the host.
+
+1. Consider partitioning the job into more tasks. This *may* result in more tasks
+generating less data each.
+
+1. Use the magic committer. This only needs enough disk storage to buffer
+blocks of the currently being written file during their upload process,
+so can use a lot less disk space.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 75c638f..aaf9f23 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -28,6 +28,8 @@ See also:
* [Encryption](./encryption.html)
* [S3Guard](./s3guard.html)
* [Troubleshooting](./troubleshooting_s3a.html)
+* [Committing work to S3 with the "S3A Committers"](./committers.html)
+* [S3A Committers Architecture](./committer_architecture.html)
* [Testing](./testing.html)
##<a name="overview"></a> Overview
@@ -82,7 +84,7 @@ the Hadoop project itself.
1. Amazon EMR's `s3://` client. This is from the Amazon EMR team, who actively
maintain it.
1. Apache's Hadoop's [`s3n:` filesystem client](./s3n.html).
- This connectore is no longer available: users must migrate to the newer `s3a:` client.
+ This connector is no longer available: users must migrate to the newer `s3a:` client.
##<a name="getting_started"></a> Getting Started
@@ -177,6 +179,7 @@ Parts of Hadoop relying on this can have unexpected behaviour. E.g. the
`AggregatedLogDeletionService` of YARN will not remove the appropriate logfiles.
* Directory listing can be slow. Use `listFiles(path, recursive)` for high
performance recursive listings whenever possible.
+* It is possible to create files under files if the caller tries hard.
* The time to rename a directory is proportional to the number of files
underneath it (directory or indirectly) and the size of the files. (The copyis
executed inside the S3 storage, so the time is independent of the bandwidth
@@ -184,8 +187,13 @@ from client to S3).
* Directory renames are not atomic: they can fail partway through, and callers
cannot safely rely on atomic renames as part of a commit algorithm.
* Directory deletion is not atomic and can fail partway through.
-* It is possible to create files under files if the caller tries hard.
+The final three issues surface when using S3 as the immediate destination
+of work, as opposed to HDFS or other "real" filesystem.
+
+The [S3A committers](./committers.html) are the sole mechanism available
+to safely save the output of queries directly into S3 object stores
+through the S3A filesystem.
### Warning #3: Object stores have differerent authorization models
@@ -223,18 +231,6 @@ Do not inadvertently share these credentials through means such as
If you do any of these: change your credentials immediately!
-### Warning #5: The S3A client cannot be used on Amazon EMR
-
-On Amazon EMR `s3a://` URLs are not supported; Amazon provide
-their own filesystem client, `s3://`.
-If you are using Amazon EMR, follow their instructions for use —and be aware
-that all issues related to S3 integration in EMR can only be addressed by Amazon
-themselves: please raise your issues with them.
-
-Equally importantly: much of this document does not apply to the EMR `s3://` client.
-Pleae consult
-[the EMR storage documentation](http://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-plan-file-systems.html)
-instead.
## <a name="authenticating"></a> Authenticating with S3
@@ -616,7 +612,7 @@ Because the provider path is not itself a sensitive secret, there is no risk
from placing its declaration on the command line.
-## <a name="general_configuration"></a>Genaral S3A Client configuration
+## <a name="general_configuration"></a>General S3A Client configuration
All S3A client options are configured with options with the prefix `fs.s3a.`.
@@ -875,6 +871,166 @@ options are covered in [Testing](./testing.md).
</property>
```
+## <a name="retry_and_recovery"></a>Retry and Recovery
+
+The S3A client makes a best-effort attempt at recovering from network failures;
+this section covers the details of what it does.
+
+The S3A divides exceptions returned by the AWS SDK into different categories,
+and chooses a differnt retry policy based on their type and whether or
+not the failing operation is idempotent.
+
+
+### Unrecoverable Problems: Fail Fast
+
+* No object/bucket store: `FileNotFoundException`
+* No access permissions: `AccessDeniedException`
+* Network errors considered unrecoverable (`UnknownHostException`,
+ `NoRouteToHostException`, `AWSRedirectException`).
+* Interruptions: `InterruptedIOException`, `InterruptedException`.
+* Rejected HTTP requests: `InvalidRequestException`
+
+These are all considered unrecoverable: S3A will make no attempt to recover
+from them.
+
+### Possibly Recoverable Problems: Retry
+
+* Connection timeout: `ConnectTimeoutException`. Timeout before
+setting up a connection to the S3 endpoint (or proxy).
+* HTTP response status code 400, "Bad Request"
+
+The status code 400, Bad Request usually means that the request
+is unrecoverable; it's the generic "No" response. Very rarely it
+does recover, which is why it is in this category, rather than that
+of unrecoverable failures.
+
+These failures will be retried with a fixed sleep interval set in
+`fs.s3a.retry.interval`, up to the limit set in `fs.s3a.retry.limit`.
+
+
+### Only retrible on idempotent operations
+
+Some network failures are considered to be retriable if they occur on
+idempotent operations; there's no way to know if they happened
+after the request was processed by S3.
+
+* `SocketTimeoutException`: general network failure.
+* `EOFException` : the connection was broken while reading data
+* "No response from Server" (443, 444) HTTP responses.
+* Any other AWS client, service or S3 exception.
+
+These failures will be retried with a fixed sleep interval set in
+`fs.s3a.retry.interval`, up to the limit set in `fs.s3a.retry.limit`.
+
+*Important*: DELETE is considered idempotent, hence: `FileSystem.delete()`
+and `FileSystem.rename()` will retry their delete requests on any
+of these failures.
+
+The issue of whether delete should be idempotent has been a source
+of historical controversy in Hadoop.
+
+1. In the absence of any other changes to the object store, a repeated
+DELETE request will eventually result in the named object being deleted;
+it's a no-op if reprocessed. As indeed, is `Filesystem.delete()`.
+1. If another client creates a file under the path, it will be deleted.
+1. Any filesystem supporting an atomic `FileSystem.create(path, overwrite=false)`
+operation to reject file creation if the path exists MUST NOT consider
+delete to be idempotent, because a `create(path, false)` operation will
+only succeed if the first `delete()` call has already succeded.
+1. And a second, retried `delete()` call could delete the new data.
+
+Because S3 is eventially consistent *and* doesn't support an
+atomic create-no-overwrite operation, the choice is more ambigious.
+
+Currently S3A considers delete to be
+idempotent because it is convenient for many workflows, including the
+commit protocols. Just be aware that in the presence of transient failures,
+more things may be deleted than expected. (For anyone who considers this to
+be the wrong decision: rebuild the `hadoop-aws` module with the constant
+`S3AFileSystem.DELETE_CONSIDERED_IDEMPOTENT` set to `false`).
+
+
+
+
+
+
+### Throttled requests from S3 and Dynamo DB
+
+
+When S3A or Dynamo DB returns a response indicating that requests
+from the caller are being throttled, an exponential back-off with
+an initial interval and a maximum number of requests.
+
+```xml
+<property>
+ <name>fs.s3a.retry.throttle.limit</name>
+ <value>${fs.s3a.attempts.maximum}</value>
+ <description>
+ Number of times to retry any throttled request.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.throttle.interval</name>
+ <value>1000ms</value>
+ <description>
+ Interval between retry attempts on throttled requests.
+ </description>
+</property>
+```
+
+Notes
+
+1. There is also throttling taking place inside the AWS SDK; this is managed
+by the value `fs.s3a.attempts.maximum`.
+1. Throttling events are tracked in the S3A filesystem metrics and statistics.
+1. Amazon KMS may thottle a customer based on the total rate of uses of
+KMS *across all user accounts and applications*.
+
+Throttling of S3 requests is all too common; it is caused by too many clients
+trying to access the same shard of S3 Storage. This generatlly
+happen if there are too many reads, those being the most common in Hadoop
+applications. This problem is exacerbated by Hive's partitioning
+strategy used when storing data, such as partitioning by year and then month.
+This results in paths with little or no variation at their start, which ends
+up in all the data being stored in the same shard(s).
+
+Here are some expensive operations; the more of these taking place
+against part of an S3 bucket, the more load it experiences.
+* Many clients trying to list directories or calling `getFileStatus` on
+paths (LIST and HEAD requests respectively)
+* The GET requests issued when reading data.
+* Random IO used when reading columnar data (ORC, Parquet) means that many
+more GET requests than a simple one-per-file read.
+* The number of active writes to that part of the S3 bucket.
+
+A special case is when enough data has been written into part of an S3 bucket
+that S3 decides to split the data across more than one shard: this
+is believed to be one by some copy operation which can take some time.
+While this is under way, S3 clients access data under these paths will
+be throttled more than usual.
+
+
+Mitigation strategies
+
+1. Use separate buckets for intermediate data/different applications/roles.
+1. Use significantly different paths for different datasets in the same bucket.
+1. Increase the value of `fs.s3a.retry.throttle.interval` to provide
+longer delays between attempts.
+1. Reduce the parallelism of the queries. The more tasks trying to access
+data in parallel, the more load.
+1. Reduce `fs.s3a.threads.max` to reduce the amount of parallel operations
+performed by clients.
+!. Maybe: increase `fs.s3a.readahead.range` to increase the minimum amount
+of data asked for in every GET request, as well as how much data is
+skipped in the existing stream before aborting it and creating a new stream.
+1. If the DynamoDB tables used by S3Guard are being throttled, increase
+the capacity through `hadoop s3guard set-capacity` (and pay more, obviously).
+1. KMS: "consult AWS about increating your capacity".
+
+
+
+
## <a name="per_bucket_configuration"></a>Configuring different S3 buckets with Per-Bucket Configuration
Different S3 buckets can be accessed with different S3A client configurations.
@@ -1081,7 +1237,7 @@ The original S3A client implemented file writes by
buffering all data to disk as it was written to the `OutputStream`.
Only when the stream's `close()` method was called would the upload start.
-This can made output slow, especially on large uploads, and could even
+This made output slow, especially on large uploads, and could even
fill up the disk space of small (virtual) disks.
Hadoop 2.7 added the `S3AFastOutputStream` alternative, which Hadoop 2.8 expanded.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
index 893f4ed..e2cb549 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/s3guard.md
@@ -426,7 +426,7 @@ hadoop s3guard diff s3a://ireland-1
Prints and optionally checks the s3guard and encryption status of a bucket.
```bash
-hadoop s3guard bucket-info [ -guarded ] [-unguarded] [-auth] [-nonauth] [-encryption ENCRYPTION] s3a://BUCKET
+hadoop s3guard bucket-info [ -guarded ] [-unguarded] [-auth] [-nonauth] [-magic] [-encryption ENCRYPTION] s3a://BUCKET
```
Options
@@ -437,6 +437,7 @@ Options
| `-unguarded` | Require S3Guard to be disabled |
| `-auth` | Require the S3Guard mode to be "authoritative" |
| `-nonauth` | Require the S3Guard mode to be "non-authoritative" |
+| `-magic` | Require the S3 filesystem to be support the "magic" committer |
| `-encryption <type>` | Require a specific server-side encryption algorithm |
The server side encryption options are not directly related to S3Guard, but
@@ -445,10 +446,11 @@ it is often convenient to check them at the same time.
Example
```bash
-hadoop s3guard bucket-info -guarded s3a://ireland-1
+hadoop s3guard bucket-info -guarded -magic s3a://ireland-1
```
List the details of bucket `s3a://ireland-1`, mandating that it must have S3Guard enabled
+("-guarded") and that support for the magic S3A committer is enabled ("-magic")
```
Filesystem s3a://ireland-1
@@ -476,6 +478,7 @@ Metadata Store Diagnostics:
TableSizeBytes: 12812,ItemCount: 91,
TableArn: arn:aws:dynamodb:eu-west-1:00000000:table/ireland-1,}
write-capacity=20
+The "magic" committer is supported
S3A Client
Endpoint: fs.s3a.endpoint=s3-eu-west-1.amazonaws.com
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
index cf7a2e4..75f8fa6 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md
@@ -629,6 +629,10 @@ Don't assume AWS S3 US-East only, do allow for working with external S3 implemen
Those may be behind the latest S3 API features, not support encryption, session
APIs, etc.
+They won't have the same CSV test files as some of the input tests rely on.
+Look at `ITestS3AInputStreamPerformance` to see how tests can be written
+to support the declaration of a specific large test file on alternate filesystems.
+
### Works Over Long-haul Links
@@ -662,6 +666,37 @@ listings, file status, ...), so help make failures easier to understand.
At the very least, do not use `assertTrue()` or `assertFalse()` without
including error messages.
+### Sets up its filesystem and checks for those settings
+
+Tests can overrun `createConfiguration()` to add new options to the configuration
+file for the S3A Filesystem instance used in their tests.
+
+However, filesystem caching may mean that a test suite may get a cached
+instance created with an differennnt configuration. For tests which don't need
+specific configurations caching is good: it reduces test setup time.
+
+For those tests which do need unique options (encryption, magic files),
+things can break, and they will do so in hard-to-replicate ways.
+
+Use `S3ATestUtils.disableFilesystemCaching(conf)` to disable caching when
+modifying the config. As an example from `AbstractTestS3AEncryption`:
+
+```java
+@Override
+protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ S3ATestUtils.disableFilesystemCaching(conf);
+ conf.set(Constants.SERVER_SIDE_ENCRYPTION_ALGORITHM,
+ getSSEAlgorithm().getMethod());
+ return conf;
+}
+```
+
+Then verify in the setup method or test cases that their filesystem actually has
+the desired feature (`fs.getConf().getProperty(...)`). This not only
+catches filesystem reuse problems, it catches the situation where the
+filesystem configuration in `auth-keys.xml` has explicit per-bucket settings
+which override the test suite's general option settings.
### Cleans Up Afterwards
@@ -677,6 +712,31 @@ get called.
We really appreciate this — you will too.
+### Runs in parallel unless this is unworkable.
+
+Tests must be designed to run in parallel with other tests, all working
+with the same shared S3 bucket. This means
+
+* Uses relative and JVM-fork-unique paths provided by the method
+ `AbstractFSContractTestBase.path(String filepath)`.
+* Doesn't manipulate the root directory or make assertions about its contents
+(for example: delete its contents and assert that it is now empty).
+* Doesn't have a specific requirement of all active clients of the bucket
+(example: SSE-C tests which require all files, even directory markers,
+to be encrypted with the same key).
+* Doesn't use so much bandwidth that all other tests will be starved of IO and
+start timing out (e.g. the scale tests).
+
+Tests such as these can only be run as sequential tests. When adding one,
+exclude it in the POM file. from the parallel failsafe run and add to the
+sequential one afterwards. The IO heavy ones must also be subclasses of
+`S3AScaleTestBase` and so only run if the system/maven property
+`fs.s3a.scale.test.enabled` is true.
+
+## Individual test cases can be run in an IDE
+
+This is invaluable for debugging test failures.
+
## <a name="tips"></a> Tips
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
index 619ffc1..7d16744 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/troubleshooting_s3a.md
@@ -721,3 +721,127 @@ http.headers (LoggingManagedHttpClientConnection.java:onResponseReceived(127)) -
http.headers (LoggingManagedHttpClientConnection.java:onResponseReceived(127)) - http-outgoing-0 << Server: AmazonS3
execchain.MainClientExec (MainClientExec.java:execute(284)) - Connection can be kept alive for 60000 MILLISECONDS
```
+
+
+## <a name="retries"></a> Reducing failures by configuring retry policy
+
+The S3A client can ba configured to rety those operations which are considered
+retriable. That can be because they are idempotent, or
+because there failure happened before the request was processed by S3.
+
+The number of retries and interval between each retry can be configured:
+
+```xml
+<property>
+ <name>fs.s3a.attempts.maximum</name>
+ <value>20</value>
+ <description>How many times we should retry commands on transient errors,
+ excluding throttling errors.</description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.interval</name>
+ <value>500ms</value>
+ <description>
+ Interval between retry attempts.
+ </description>
+</property>
+```
+
+Not all failures are retried. Specifically excluded are those considered
+unrecoverable:
+
+* Low-level networking: `UnknownHostException`, `NoRouteToHostException`.
+* 302 redirects
+* Missing resources, 404/`FileNotFoundException`
+* HTTP 416 response/`EOFException`. This can surface if the length of a file changes
+ while another client is reading it.
+* Failures during execution or result processing of non-idempotent operations where
+it is considered likely that the operation has already taken place.
+
+In future, others may be added to this list.
+
+When one of these failures arises in the S3/S3A client, the retry mechanism
+is bypassed and the operation will fail.
+
+*Warning*: the S3A client considers DELETE, PUT and COPY operations to
+be idempotent, and will retry them on failure. These are only really idempotent
+if no other client is attempting to manipulate the same objects, such as:
+renaming() the directory tree or uploading files to the same location.
+Please don't do that. Given that the emulated directory rename and delete operations
+aren't atomic, even without retries, multiple S3 clients working with the same
+paths can interfere with each other
+
+#### <a name="retries"></a> Throttling
+
+When many requests are made of a specific S3 bucket (or shard inside it),
+S3 will respond with a 503 "throttled" response.
+Throttling can be recovered from, provided overall load decreases.
+Furthermore, because it is sent before any changes are made to the object store,
+is inherently idempotent. For this reason, the client will always attempt to
+retry throttled requests.
+
+The limit of the number of times a throttled request can be retried,
+and the exponential interval increase between attempts, can be configured
+independently of the other retry limits.
+
+```xml
+<property>
+ <name>fs.s3a.retry.throttle.limit</name>
+ <value>20</value>
+ <description>
+ Number of times to retry any throttled request.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.retry.throttle.interval</name>
+ <value>500ms</value>
+ <description>
+ Interval between retry attempts on throttled requests.
+ </description>
+</property>
+```
+
+If a client is failing due to `AWSServiceThrottledException` failures,
+increasing the interval and limit *may* address this. However, it
+it is a sign of AWS services being overloaded by the sheer number of clients
+and rate of requests. Spreading data across different buckets, and/or using
+a more balanced directory structure may be beneficial.
+Consult [the AWS documentation](http://docs.aws.amazon.com/AmazonS3/latest/dev/request-rate-perf-considerations.html).
+
+Reading or writing data encrypted with SSE-KMS forces S3 to make calls of
+the AWS KMS Key Management Service, which comes with its own
+[Request Rate Limits](http://docs.aws.amazon.com/kms/latest/developerguide/limits.html).
+These default to 1200/second for an account, across all keys and all uses of
+them, which, for S3 means: across all buckets with data encrypted with SSE-KMS.
+
+###### Tips to Keep Throttling down
+
+* If you are seeing a lot of throttling responses on a large scale
+operation like a `distcp` copy, *reduce* the number of processes trying
+to work with the bucket (for distcp: reduce the number of mappers with the
+`-m` option).
+
+* If you are reading or writing lists of files, if you can randomize
+the list so they are not processed in a simple sorted order, you may
+reduce load on a specific shard of S3 data, so potentially increase throughput.
+
+* An S3 Bucket is throttled by requests coming from all
+simultaneous clients. Different applications and jobs may interfere with
+each other: consider that when troubleshooting.
+Partitioning data into different buckets may help isolate load here.
+
+* If you are using data encrypted with SSE-KMS, then the
+will also apply: these are stricter than the S3 numbers.
+If you believe that you are reaching these limits, you may be able to
+get them increased.
+Consult [the KMS Rate Limit documentation](http://docs.aws.amazon.com/kms/latest/developerguide/limits.html).
+
+* S3Guard uses DynamoDB for directory and file lookups;
+it is rate limited to the amount of (guaranteed) IO purchased for a
+table. If significant throttling events/rate is observed here, the preallocated
+IOPs can be increased with the `s3guard set-capacity` command, or
+through the AWS Console. Throttling events in S3Guard are noted in logs, and
+also in the S3A metrics `s3guard_metadatastore_throttle_rate` and
+`s3guard_metadatastore_throttled`.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
index cb9819c..c2dc700 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractGetFileStatus.java
@@ -21,12 +21,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.contract.AbstractFSContract;
import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3ATestConstants;
import org.apache.hadoop.fs.s3a.S3ATestUtils;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
/**
* S3A contract tests covering getFileStatus.
+ * Some of the tests can take too long when the fault injection rate is high,
+ * so the test timeout is extended.
*/
public class ITestS3AContractGetFileStatus
extends AbstractContractGetFileStatusTest {
@@ -52,4 +55,12 @@ public class ITestS3AContractGetFileStatus
maybeEnableS3Guard(conf);
return conf;
}
+
+ /**
+ * {@inheritDoc}
+ * @return S3A test timeout.
+ */
+ protected int getTestTimeoutMillis() {
+ return S3ATestConstants.S3A_TEST_TIMEOUT;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
index 0c7f7df..03c91e6 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -26,6 +26,7 @@ import com.amazonaws.services.s3.AmazonS3;
import java.net.URI;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
import org.apache.hadoop.fs.s3a.s3guard.NullMetadataStore;
@@ -62,10 +63,12 @@ public abstract class AbstractS3AMockTest {
// test we don't issue request to AWS DynamoDB service.
conf.setClass(S3_METADATA_STORE_IMPL, NullMetadataStore.class,
MetadataStore.class);
+ // FS is always magic
+ conf.setBoolean(CommitConstants.MAGIC_COMMITTER_ENABLED, true);
fs = new S3AFileSystem();
URI uri = URI.create(FS_S3A + "://" + BUCKET);
fs.initialize(uri, conf);
- s3 = fs.getAmazonS3Client();
+ s3 = fs.getAmazonS3ClientForTesting("mocking");
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
index f0c389d..73e71f4 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3ATestBase.java
@@ -29,11 +29,13 @@ import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
import static org.apache.hadoop.fs.contract.ContractTestUtils.writeDataset;
import static org.apache.hadoop.fs.s3a.S3ATestUtils.maybeEnableS3Guard;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
/**
* An extension of the contract test base set up for S3A tests.
@@ -58,7 +60,11 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
@Before
public void nameThread() {
- Thread.currentThread().setName("JUnit-" + methodName.getMethodName());
+ Thread.currentThread().setName("JUnit-" + getMethodName());
+ }
+
+ protected String getMethodName() {
+ return methodName.getMethodName();
}
@Override
@@ -75,6 +81,19 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
Configuration conf = super.createConfiguration();
// patch in S3Guard options
maybeEnableS3Guard(conf);
+ // set hadoop temp dir to a default value
+ String testUniqueForkId =
+ System.getProperty(TEST_UNIQUE_FORK_ID);
+ String tmpDir = conf.get(Constants.HADOOP_TMP_DIR, "target/build/test");
+ if (testUniqueForkId != null) {
+ // patch temp dir for the specific branch
+ tmpDir = tmpDir + File.pathSeparatorChar + testUniqueForkId;
+ conf.set(Constants.HADOOP_TMP_DIR, tmpDir);
+ }
+ conf.set(Constants.BUFFER_DIR, tmpDir);
+ // add this so that even on tests where the FS is shared,
+ // the FS is always "magic"
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
return conf;
}
@@ -98,7 +117,7 @@ public abstract class AbstractS3ATestBase extends AbstractFSContractTestBase
*/
protected void describe(String text, Object... args) {
LOG.info("\n\n{}: {}\n",
- methodName.getMethodName(),
+ getMethodName(),
String.format(text, args));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index b38f191..f1799ac 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -110,7 +110,7 @@ public class ITestS3AConfiguration {
} else {
conf.set(Constants.ENDPOINT, endpoint);
fs = S3ATestUtils.createTestFileSystem(conf);
- AmazonS3 s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3ClientForTesting("test endpoint");
String endPointRegion = "";
// Differentiate handling of "s3-" and "s3." based endpoint identifiers
String[] endpointParts = StringUtils.split(endpoint, '.');
@@ -378,7 +378,7 @@ public class ITestS3AConfiguration {
try {
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3 s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3ClientForTesting("configuration");
assertNotNull(s3);
S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
"clientOptions");
@@ -402,7 +402,7 @@ public class ITestS3AConfiguration {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3 s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3ClientForTesting("User Agent");
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
@@ -416,7 +416,7 @@ public class ITestS3AConfiguration {
conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3 s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3ClientForTesting("User agent");
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
index 8b7e031..bac1bb1 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AEncryptionSSEC.java
@@ -222,11 +222,10 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
//unencrypted can access until the final directory
unencryptedFileSystem.listFiles(pathA, true);
unencryptedFileSystem.listFiles(pathAB, true);
- AWSS3IOException ex = intercept(AWSS3IOException.class,
+ AWSBadRequestException ex = intercept(AWSBadRequestException.class,
() -> {
unencryptedFileSystem.listFiles(pathABC, false);
});
- assertStatusCode(ex, 400);
}
/**
@@ -270,11 +269,10 @@ public class ITestS3AEncryptionSSEC extends AbstractTestS3AEncryption {
unencryptedFileSystem.listStatus(pathA);
unencryptedFileSystem.listStatus(pathAB);
- AWSS3IOException ex = intercept(AWSS3IOException.class,
+ intercept(AWSBadRequestException.class,
() -> {
unencryptedFileSystem.listStatus(pathABC);
});
- assertStatusCode(ex, 400);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
index 7d03668..5cd7379 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFailureHandling.java
@@ -23,9 +23,7 @@ import com.amazonaws.services.s3.model.MultiObjectDeleteException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.InvalidRequestException;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Assume;
import org.junit.Test;
@@ -34,11 +32,12 @@ import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.FileNotFoundException;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.Callable;
import static org.apache.hadoop.fs.contract.ContractTestUtils.*;
+import static org.apache.hadoop.test.LambdaTestUtils.*;
/**
* Test S3A Failure translation, including a functional test
@@ -73,13 +72,9 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
writeDataset(fs, testpath, shortDataset, shortDataset.length, 1024, true);
// here the file length is less. Probe the file to see if this is true,
// with a spin and wait
- LambdaTestUtils.eventually(30 * 1000, 1000,
- new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
- return null;
- }
+ eventually(30 * 1000, 1000,
+ () -> {
+ assertEquals(shortLen, fs.getFileStatus(testpath).getLen());
});
// here length is shorter. Assuming it has propagated to all replicas,
@@ -99,12 +94,8 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
instream.read(instream.getPos(), buf, 0, buf.length));
// now do a block read fully, again, backwards from the current pos
- try {
- instream.readFully(shortLen + 512, buf);
- fail("Expected readFully to fail");
- } catch (EOFException expected) {
- LOG.debug("Expected EOF: ", expected);
- }
+ intercept(EOFException.class, "", "readfully",
+ () -> instream.readFully(shortLen + 512, buf));
assertIsEOF("read(offset)",
instream.read(shortLen + 510, buf, 0, buf.length));
@@ -115,19 +106,10 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
// delete the file. Reads must fail
fs.delete(testpath, false);
- try {
- int r = instream.read();
- fail("Expected an exception, got " + r);
- } catch (FileNotFoundException e) {
- // expected
- }
-
- try {
- instream.readFully(2048, buf);
- fail("Expected readFully to fail");
- } catch (FileNotFoundException e) {
- // expected
- }
+ intercept(FileNotFoundException.class, "", "read()",
+ () -> instream.read());
+ intercept(FileNotFoundException.class, "", "readfully",
+ () -> instream.readFully(2048, buf));
}
}
@@ -149,7 +131,7 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
}
private void removeKeys(S3AFileSystem fileSystem, String... keys)
- throws InvalidRequestException {
+ throws IOException {
List<DeleteObjectsRequest.KeyVersion> request = new ArrayList<>(
keys.length);
for (String key : keys) {
@@ -168,14 +150,15 @@ public class ITestS3AFailureHandling extends AbstractS3ATestBase {
timer.end("removeKeys");
}
- @Test(expected = MultiObjectDeleteException.class)
+ @Test
public void testMultiObjectDeleteNoPermissions() throws Throwable {
Configuration conf = getConfiguration();
String csvFile = conf.getTrimmed(KEY_CSVTEST_FILE, DEFAULT_CSVTEST_FILE);
Assume.assumeTrue("CSV test file is not the default",
DEFAULT_CSVTEST_FILE.equals(csvFile));
Path testFile = new Path(csvFile);
- S3AFileSystem fs = (S3AFileSystem)FileSystem.newInstance(testFile.toUri(), conf);
- removeKeys(fs, fs.pathToKey(testFile));
+ S3AFileSystem fs = (S3AFileSystem)testFile.getFileSystem(conf);
+ intercept(MultiObjectDeleteException.class,
+ () -> removeKeys(fs, fs.pathToKey(testFile)));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
index 3e293f7..e56fdf8 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AFileOperationCost.java
@@ -203,6 +203,8 @@ public class ITestS3AFileOperationCost extends AbstractS3ATestBase {
// before the internal behavior w/ or w/o metadata store.
assumeFalse(fs.hasMetadataStore());
+ skipDuringFaultInjection(fs);
+
Path srcBaseDir = path("src");
mkdirs(srcBaseDir);
MetricDiff deleteRequests =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
index 4abe2b7..44a2beb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3ATemporaryCredentials.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.fs.s3a;
import java.io.IOException;
import java.net.URI;
-import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
import com.amazonaws.services.securitytoken.model.GetSessionTokenRequest;
@@ -30,6 +29,7 @@ import com.amazonaws.services.securitytoken.model.Credentials;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.LambdaTestUtils;
import org.junit.Test;
@@ -113,7 +113,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase {
createAndVerifyFile(fs, path("testSTSInvalidToken"), TEST_FILE_SIZE);
fail("Expected an access exception, but file access to "
+ fs.getUri() + " was allowed: " + fs);
- } catch (AWSS3IOException ex) {
+ } catch (AWSS3IOException | AWSBadRequestException ex) {
LOG.info("Expected Exception: {}", ex.toString());
LOG.debug("Expected Exception: {}", ex, ex);
}
@@ -125,14 +125,7 @@ public class ITestS3ATemporaryCredentials extends AbstractS3ATestBase {
conf.set(ACCESS_KEY, "accesskey");
conf.set(SECRET_KEY, "secretkey");
conf.set(SESSION_TOKEN, "");
- TemporaryAWSCredentialsProvider provider
- = new TemporaryAWSCredentialsProvider(conf);
- try {
- AWSCredentials credentials = provider.getCredentials();
- fail("Expected a CredentialInitializationException,"
- + " got " + credentials);
- } catch (CredentialInitializationException expected) {
- // expected
- }
+ LambdaTestUtils.intercept(CredentialInitializationException.class,
+ () -> new TemporaryAWSCredentialsProvider(conf).getCredentials());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
index bc03a17..4a81374 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3GuardListConsistency.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.s3a;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.AmazonS3;
import org.apache.hadoop.conf.Configuration;
@@ -52,6 +53,16 @@ import static org.apache.hadoop.fs.s3a.InconsistentAmazonS3Client.*;
*/
public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
+ private Invoker invoker;
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ invoker = new Invoker(new S3ARetryPolicy(getConfiguration()),
+ Invoker.NO_OP
+ );
+ }
+
@Override
protected AbstractFSContract createContract(Configuration conf) {
conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
@@ -503,20 +514,15 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
}
clearInconsistency(fs);
- AmazonS3 client = fs.getAmazonS3Client();
String key = fs.pathToKey(root) + "/";
- ListObjectsV2Result preDeleteDelimited = client.listObjectsV2(
- fs.createListObjectsRequest(key, "/").getV2());
- ListObjectsV2Result preDeleteUndelimited = client.listObjectsV2(
- fs.createListObjectsRequest(key, null).getV2());
+ ListObjectsV2Result preDeleteDelimited = listObjectsV2(fs, key, "/");
+ ListObjectsV2Result preDeleteUndelimited = listObjectsV2(fs, key, null);
fs.delete(root, true);
- ListObjectsV2Result postDeleteDelimited = client.listObjectsV2(
- fs.createListObjectsRequest(key, "/").getV2());
- ListObjectsV2Result postDeleteUndelimited = client.listObjectsV2(
- fs.createListObjectsRequest(key, null).getV2());
+ ListObjectsV2Result postDeleteDelimited = listObjectsV2(fs, key, "/");
+ ListObjectsV2Result postDeleteUndelimited = listObjectsV2(fs, key, null);
assertEquals("InconsistentAmazonS3Client added back objects incorrectly " +
"in a non-recursive listing",
@@ -540,8 +546,27 @@ public class ITestS3GuardListConsistency extends AbstractS3ATestBase {
);
}
+ /**
+ * retrying v2 list.
+ * @param fs
+ * @param key
+ * @param delimiter
+ * @return
+ * @throws IOException
+ */
+
+ private ListObjectsV2Result listObjectsV2(S3AFileSystem fs,
+ String key, String delimiter) throws java.io.IOException {
+ ListObjectsV2Request k = fs.createListObjectsRequest(key, delimiter)
+ .getV2();
+ return invoker.retryUntranslated("list", true,
+ () -> {
+ return fs.getAmazonS3ClientForTesting("list").listObjectsV2(k);
+ });
+ }
+
private static void clearInconsistency(S3AFileSystem fs) throws Exception {
- AmazonS3 s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3ClientForTesting("s3guard");
InconsistentAmazonS3Client ic = InconsistentAmazonS3Client.castFrom(s3);
ic.clearInconsistency();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/49] hadoop git commit: HDFS-12813. RequestHedgingProxyProvider
can hide Exception thrown from the Namenode for proxy size of 1. Contributed
by Mukul Kumar Singh
Posted by su...@apache.org.
HDFS-12813. RequestHedgingProxyProvider can hide Exception thrown from the Namenode for proxy size of 1. Contributed by Mukul Kumar Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/659e85e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/659e85e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/659e85e3
Branch: refs/heads/YARN-5881
Commit: 659e85e304d070f9908a96cf6a0e1cbafde6a434
Parents: 60fc2a1
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Mon Nov 20 17:09:19 2017 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Mon Nov 20 17:09:19 2017 -0800
----------------------------------------------------------------------
.../ha/RequestHedgingProxyProvider.java | 81 ++++++++++++++------
.../ha/TestRequestHedgingProxyProvider.java | 58 ++++++++++++++
2 files changed, 114 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/659e85e3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
index b94e94d..08edfe2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/RequestHedgingProxyProvider.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.ha;
import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.net.URI;
@@ -29,6 +30,7 @@ import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ExecutionException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RemoteException;
@@ -87,9 +89,19 @@ public class RequestHedgingProxyProvider<T> extends
targetProxies.remove(toIgnore);
if (targetProxies.size() == 1) {
ProxyInfo<T> proxyInfo = targetProxies.values().iterator().next();
- Object retVal = method.invoke(proxyInfo.proxy, args);
- successfulProxy = proxyInfo;
- return retVal;
+ try {
+ currentUsedProxy = proxyInfo;
+ Object retVal = method.invoke(proxyInfo.proxy, args);
+ LOG.debug("Invocation successful on [{}]",
+ currentUsedProxy.proxyInfo);
+ return retVal;
+ } catch (InvocationTargetException ex) {
+ Exception unwrappedException = unwrapInvocationTargetException(ex);
+ logProxyException(unwrappedException, currentUsedProxy.proxyInfo);
+ LOG.trace("Unsuccessful invocation on [{}]",
+ currentUsedProxy.proxyInfo);
+ throw unwrappedException;
+ }
}
executor = Executors.newFixedThreadPool(proxies.size());
completionService = new ExecutorCompletionService<>(executor);
@@ -112,15 +124,16 @@ public class RequestHedgingProxyProvider<T> extends
Future<Object> callResultFuture = completionService.take();
Object retVal;
try {
+ currentUsedProxy = proxyMap.get(callResultFuture);
retVal = callResultFuture.get();
- successfulProxy = proxyMap.get(callResultFuture);
LOG.debug("Invocation successful on [{}]",
- successfulProxy.proxyInfo);
+ currentUsedProxy.proxyInfo);
return retVal;
- } catch (Exception ex) {
+ } catch (ExecutionException ex) {
+ Exception unwrappedException = unwrapExecutionException(ex);
ProxyInfo<T> tProxyInfo = proxyMap.get(callResultFuture);
- logProxyException(ex, tProxyInfo.proxyInfo);
- badResults.put(tProxyInfo.proxyInfo, unwrapException(ex));
+ logProxyException(unwrappedException, tProxyInfo.proxyInfo);
+ badResults.put(tProxyInfo.proxyInfo, unwrappedException);
LOG.trace("Unsuccessful invocation on [{}]", tProxyInfo.proxyInfo);
numAttempts--;
}
@@ -143,7 +156,7 @@ public class RequestHedgingProxyProvider<T> extends
}
- private volatile ProxyInfo<T> successfulProxy = null;
+ private volatile ProxyInfo<T> currentUsedProxy = null;
private volatile String toIgnore = null;
public RequestHedgingProxyProvider(Configuration conf, URI uri,
@@ -154,8 +167,8 @@ public class RequestHedgingProxyProvider<T> extends
@SuppressWarnings("unchecked")
@Override
public synchronized ProxyInfo<T> getProxy() {
- if (successfulProxy != null) {
- return successfulProxy;
+ if (currentUsedProxy != null) {
+ return currentUsedProxy;
}
Map<String, ProxyInfo<T>> targetProxyInfos = new HashMap<>();
StringBuilder combinedInfo = new StringBuilder("[");
@@ -175,8 +188,8 @@ public class RequestHedgingProxyProvider<T> extends
@Override
public synchronized void performFailover(T currentProxy) {
- toIgnore = successfulProxy.proxyInfo;
- successfulProxy = null;
+ toIgnore = this.currentUsedProxy.proxyInfo;
+ this.currentUsedProxy = null;
}
/**
@@ -187,19 +200,18 @@ public class RequestHedgingProxyProvider<T> extends
*/
private void logProxyException(Exception ex, String proxyInfo) {
if (isStandbyException(ex)) {
- LOG.debug("Invocation returned standby exception on [{}]", proxyInfo);
+ LOG.debug("Invocation returned standby exception on [{}]", proxyInfo, ex);
} else {
- LOG.warn("Invocation returned exception on [{}]", proxyInfo);
+ LOG.warn("Invocation returned exception on [{}]", proxyInfo, ex);
}
}
/**
* Check if the returned exception is caused by an standby namenode.
- * @param ex Exception to check.
+ * @param exception Exception to check.
* @return If the exception is caused by an standby namenode.
*/
- private boolean isStandbyException(Exception ex) {
- Exception exception = unwrapException(ex);
+ private boolean isStandbyException(Exception exception) {
if (exception instanceof RemoteException) {
return ((RemoteException) exception).unwrapRemoteException()
instanceof StandbyException;
@@ -208,24 +220,43 @@ public class RequestHedgingProxyProvider<T> extends
}
/**
- * Unwraps the exception. <p>
+ * Unwraps the ExecutionException. <p>
* Example:
* <blockquote><pre>
* if ex is
- * ExecutionException(InvocationTargetExeption(SomeException))
+ * ExecutionException(InvocationTargetException(SomeException))
* returns SomeException
* </pre></blockquote>
*
* @return unwrapped exception
*/
- private Exception unwrapException(Exception ex) {
+ private Exception unwrapExecutionException(ExecutionException ex) {
+ if (ex != null) {
+ Throwable cause = ex.getCause();
+ if (cause instanceof InvocationTargetException) {
+ return
+ unwrapInvocationTargetException((InvocationTargetException)cause);
+ }
+ }
+ return ex;
+
+ }
+
+ /**
+ * Unwraps the InvocationTargetException. <p>
+ * Example:
+ * <blockquote><pre>
+ * if ex is InvocationTargetException(SomeException)
+ * returns SomeException
+ * </pre></blockquote>
+ *
+ * @return unwrapped exception
+ */
+ private Exception unwrapInvocationTargetException(
+ InvocationTargetException ex) {
if (ex != null) {
Throwable cause = ex.getCause();
if (cause instanceof Exception) {
- Throwable innerCause = cause.getCause();
- if (innerCause instanceof Exception) {
- return (Exception) innerCause;
- }
return (Exception) cause;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/659e85e3/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
index 724b5f0..8feba96 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestRequestHedgingProxyProvider.java
@@ -232,6 +232,64 @@ public class TestRequestHedgingProxyProvider {
}
@Test
+ public void testFileNotFoundExceptionWithSingleProxy() throws Exception {
+ ClientProtocol active = Mockito.mock(ClientProtocol.class);
+ Mockito
+ .when(active.getBlockLocations(Matchers.anyString(),
+ Matchers.anyLong(), Matchers.anyLong()))
+ .thenThrow(new RemoteException("java.io.FileNotFoundException",
+ "File does not exist!"));
+
+ ClientProtocol standby = Mockito.mock(ClientProtocol.class);
+ Mockito
+ .when(standby.getBlockLocations(Matchers.anyString(),
+ Matchers.anyLong(), Matchers.anyLong()))
+ .thenThrow(
+ new RemoteException("org.apache.hadoop.ipc.StandbyException",
+ "Standby NameNode"));
+
+ RequestHedgingProxyProvider<ClientProtocol> provider =
+ new RequestHedgingProxyProvider<>(conf, nnUri,
+ ClientProtocol.class, createFactory(standby, active));
+ try {
+ provider.getProxy().proxy.getBlockLocations("/tmp/test.file", 0L, 20L);
+ Assert.fail("Should fail since the active namenode throws"
+ + " FileNotFoundException!");
+ } catch (MultiException me) {
+ for (Exception ex : me.getExceptions().values()) {
+ Exception rEx = ((RemoteException) ex).unwrapRemoteException();
+ if (rEx instanceof StandbyException) {
+ continue;
+ }
+ Assert.assertTrue(rEx instanceof FileNotFoundException);
+ }
+ }
+ //Perform failover now, there will only be one active proxy now
+ provider.performFailover(active);
+ try {
+ provider.getProxy().proxy.getBlockLocations("/tmp/test.file", 0L, 20L);
+ Assert.fail("Should fail since the active namenode throws"
+ + " FileNotFoundException!");
+ } catch (RemoteException ex) {
+ Exception rEx = ex.unwrapRemoteException();
+ if (rEx instanceof StandbyException) {
+ Mockito.verify(active).getBlockLocations(Matchers.anyString(),
+ Matchers.anyLong(), Matchers.anyLong());
+ Mockito.verify(standby, Mockito.times(2))
+ .getBlockLocations(Matchers.anyString(),
+ Matchers.anyLong(), Matchers.anyLong());
+ } else {
+ Assert.assertTrue(rEx instanceof FileNotFoundException);
+ Mockito.verify(active, Mockito.times(2))
+ .getBlockLocations(Matchers.anyString(),
+ Matchers.anyLong(), Matchers.anyLong());
+ Mockito.verify(standby).getBlockLocations(Matchers.anyString(),
+ Matchers.anyLong(), Matchers.anyLong());
+ }
+ }
+ }
+
+ @Test
public void testPerformFailoverWith3Proxies() throws Exception {
conf.set(HdfsClientConfigKeys.DFS_HA_NAMENODES_KEY_PREFIX + "." + ns,
"nn1,nn2,nn3");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitterFactory.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitterFactory.java
new file mode 100644
index 0000000..13e1c61
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/lib/output/TestPathOutputCommitterFactory.java
@@ -0,0 +1,495 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce.lib.output;
+
+import java.io.IOException;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+
+import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test the committer factory logic, looking at the override
+ * and fallback behavior.
+ */
+@SuppressWarnings("unchecked")
+public class TestPathOutputCommitterFactory extends Assert {
+
+ private static final String HTTP_COMMITTER_FACTORY = String.format(
+ COMMITTER_FACTORY_SCHEME_PATTERN, "http");
+
+ private static final Path HTTP_PATH = new Path("http://hadoop.apache.org/");
+ private static final Path HDFS_PATH = new Path("hdfs://localhost:8081/");
+
+ private TaskAttemptID taskAttemptID =
+ new TaskAttemptID("local", 0, TaskType.MAP, 1, 2);
+
+ /**
+ * Set a factory for a schema, verify it works.
+ * @throws Throwable failure
+ */
+ @Test
+ public void testCommitterFactoryForSchema() throws Throwable {
+ createCommitterFactory(SimpleCommitterFactory.class,
+ HTTP_PATH,
+ newBondedConfiguration());
+ }
+
+ /**
+ * A schema factory only affects that filesystem.
+ * @throws Throwable failure
+ */
+ @Test
+ public void testCommitterFactoryFallbackDefault() throws Throwable {
+ createCommitterFactory(FileOutputCommitterFactory.class,
+ HDFS_PATH,
+ newBondedConfiguration());
+ }
+
+ /**
+ * A schema factory only affects that filesystem; test through
+ * {@link PathOutputCommitterFactory#createCommitter(Path, TaskAttemptContext)}.
+ * @throws Throwable failure
+ */
+ @Test
+ public void testCommitterFallbackDefault() throws Throwable {
+ createCommitter(FileOutputCommitter.class,
+ HDFS_PATH,
+ taskAttempt(newBondedConfiguration()));
+ }
+
+ /**
+ * Verify that you can override any schema with an explicit name.
+ */
+ @Test
+ public void testCommitterFactoryOverride() throws Throwable {
+ Configuration conf = newBondedConfiguration();
+ // set up for the schema factory
+ // and then set a global one which overrides the others.
+ conf.set(COMMITTER_FACTORY_CLASS, OtherFactory.class.getName());
+ createCommitterFactory(OtherFactory.class, HDFS_PATH, conf);
+ createCommitterFactory(OtherFactory.class, HTTP_PATH, conf);
+ }
+
+ /**
+ * Verify that if the factory class option is "", schema factory
+ * resolution still works.
+ */
+ @Test
+ public void testCommitterFactoryEmptyOption() throws Throwable {
+ Configuration conf = newBondedConfiguration();
+ // set up for the schema factory
+ // and then set a global one which overrides the others.
+ conf.set(COMMITTER_FACTORY_CLASS, "");
+ createCommitterFactory(SimpleCommitterFactory.class, HTTP_PATH, conf);
+
+ // and HDFS, with no schema, falls back to the default
+ createCommitterFactory(FileOutputCommitterFactory.class, HDFS_PATH, conf);
+ }
+
+ /**
+ * Verify that if the committer factory class is unknown, you cannot
+ * create committers.
+ */
+ @Test
+ public void testCommitterFactoryUnknown() throws Throwable {
+ Configuration conf = new Configuration();
+ // set the factory to an unknown class
+ conf.set(COMMITTER_FACTORY_CLASS, "unknown");
+ intercept(RuntimeException.class,
+ () -> getCommitterFactory(HDFS_PATH, conf));
+ }
+
+ /**
+ * Verify that if the committer output path is null, you get back
+ * a FileOutputCommitter with null output & work paths.
+ */
+ @Test
+ public void testCommitterNullOutputPath() throws Throwable {
+ // bind http to schema
+ Configuration conf = newBondedConfiguration();
+ // then ask committers for a null path
+ FileOutputCommitter committer = createCommitter(
+ FileOutputCommitterFactory.class,
+ FileOutputCommitter.class,
+ null, conf);
+ assertNull(committer.getOutputPath());
+ assertNull(committer.getWorkPath());
+ }
+
+ /**
+ * Verify that if you explicitly name a committer, that takes priority
+ * over any filesystem committer.
+ */
+ @Test
+ public void testNamedCommitterFactory() throws Throwable {
+ Configuration conf = new Configuration();
+ // set up for the schema factory
+ conf.set(COMMITTER_FACTORY_CLASS, NAMED_COMMITTER_FACTORY);
+ conf.set(NAMED_COMMITTER_CLASS, SimpleCommitter.class.getName());
+ SimpleCommitter sc = createCommitter(
+ NamedCommitterFactory.class,
+ SimpleCommitter.class, HDFS_PATH, conf);
+ assertEquals("Wrong output path from " + sc,
+ HDFS_PATH,
+ sc.getOutputPath());
+ }
+
+ /**
+ * Verify that if you explicitly name a committer and there's no
+ * path, the committer is picked up.
+ */
+ @Test
+ public void testNamedCommitterFactoryNullPath() throws Throwable {
+ Configuration conf = new Configuration();
+ // set up for the schema factory
+ conf.set(COMMITTER_FACTORY_CLASS, NAMED_COMMITTER_FACTORY);
+ conf.set(NAMED_COMMITTER_CLASS, SimpleCommitter.class.getName());
+ SimpleCommitter sc = createCommitter(
+ NamedCommitterFactory.class,
+ SimpleCommitter.class,
+ null, conf);
+ assertNull(sc.getOutputPath());
+ }
+
+ /**
+ * Verify that if you explicitly name a committer and there's no
+ * path, the committer is picked up.
+ */
+ @Test
+ public void testNamedCommitterNullPath() throws Throwable {
+ Configuration conf = new Configuration();
+ // set up for the schema factory
+ conf.set(COMMITTER_FACTORY_CLASS, NAMED_COMMITTER_FACTORY);
+ conf.set(NAMED_COMMITTER_CLASS, SimpleCommitter.class.getName());
+
+ SimpleCommitter sc = createCommitter(
+ SimpleCommitter.class,
+ null, taskAttempt(conf));
+ assertNull(sc.getOutputPath());
+ }
+
+ /**
+ * Create a factory then a committer, validating the type of both.
+ * @param <T> type of factory
+ * @param <U> type of committer
+ * @param factoryClass expected factory class
+ * @param committerClass expected committer class
+ * @param path output path (may be null)
+ * @param conf configuration
+ * @return the committer
+ * @throws IOException failure to create
+ */
+ private <T extends PathOutputCommitterFactory, U extends PathOutputCommitter>
+ U createCommitter(Class<T> factoryClass,
+ Class<U> committerClass,
+ Path path,
+ Configuration conf) throws IOException {
+ T f = createCommitterFactory(factoryClass, path, conf);
+ PathOutputCommitter committer = f.createOutputCommitter(path,
+ taskAttempt(conf));
+ assertEquals(" Wrong committer for path " + path + " from factory " + f,
+ committerClass, committer.getClass());
+ return (U) committer;
+ }
+
+ /**
+ * Create a committer from a task context, via
+ * {@link PathOutputCommitterFactory#createCommitter(Path, TaskAttemptContext)}.
+ * @param <U> type of committer
+ * @param committerClass expected committer class
+ * @param path output path (may be null)
+ * @param context task attempt context
+ * @return the committer
+ * @throws IOException failure to create
+ */
+ private <U extends PathOutputCommitter> U createCommitter(
+ Class<U> committerClass,
+ Path path,
+ TaskAttemptContext context) throws IOException {
+ PathOutputCommitter committer = PathOutputCommitterFactory
+ .createCommitter(path, context);
+ assertEquals(" Wrong committer for path " + path,
+ committerClass, committer.getClass());
+ return (U) committer;
+ }
+
+ /**
+ * Create a factory then a committer, validating its type.
+ * @param factoryClass expected factory class
+ * @param path output path (may be null)
+ * @param conf configuration
+ * @param <T> type of factory
+ * @return the factory
+ */
+ private <T extends PathOutputCommitterFactory> T createCommitterFactory(
+ Class<T> factoryClass,
+ Path path,
+ Configuration conf) {
+ PathOutputCommitterFactory factory = getCommitterFactory(path, conf);
+ assertEquals(" Wrong factory for path " + path,
+ factoryClass, factory.getClass());
+ return (T)factory;
+ }
+
+ /**
+ * Create a new task attempt context.
+ * @param conf config
+ * @return a new context
+ */
+ private TaskAttemptContext taskAttempt(Configuration conf) {
+ return new TaskAttemptContextImpl(conf, taskAttemptID);
+ }
+
+ /**
+ * Verify that if you explicitly name a committer, that takes priority
+ * over any filesystem committer.
+ */
+ @Test
+ public void testFileOutputCommitterFactory() throws Throwable {
+ Configuration conf = new Configuration();
+ // set up for the schema factory
+ conf.set(COMMITTER_FACTORY_CLASS, FILE_COMMITTER_FACTORY);
+ conf.set(NAMED_COMMITTER_CLASS, SimpleCommitter.class.getName());
+ getCommitterFactory(HDFS_PATH, conf);
+ createCommitter(
+ FileOutputCommitterFactory.class,
+ FileOutputCommitter.class, null, conf);
+ }
+
+ /**
+ * Follow the entire committer chain down and create a new committer from
+ * the output format.
+ * @throws Throwable on a failure.
+ */
+ @Test
+ public void testFileOutputFormatBinding() throws Throwable {
+ Configuration conf = newBondedConfiguration();
+ conf.set(FileOutputFormat.OUTDIR, HTTP_PATH.toUri().toString());
+ TextOutputFormat<String, String> off = new TextOutputFormat<>();
+ SimpleCommitter committer = (SimpleCommitter)
+ off.getOutputCommitter(taskAttempt(conf));
+ assertEquals("Wrong output path from "+ committer,
+ HTTP_PATH,
+ committer.getOutputPath());
+ }
+
+ /**
+ * Follow the entire committer chain down and create a new committer from
+ * the output format.
+ * @throws Throwable on a failure.
+ */
+ @Test
+ public void testFileOutputFormatBindingNoPath() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.unset(FileOutputFormat.OUTDIR);
+ // set up for the schema factory
+ conf.set(COMMITTER_FACTORY_CLASS, NAMED_COMMITTER_FACTORY);
+ conf.set(NAMED_COMMITTER_CLASS, SimpleCommitter.class.getName());
+ httpToSimpleFactory(conf);
+ TextOutputFormat<String, String> off = new TextOutputFormat<>();
+ SimpleCommitter committer = (SimpleCommitter)
+ off.getOutputCommitter(taskAttempt(conf));
+ assertNull("Output path from "+ committer,
+ committer.getOutputPath());
+ }
+
+ /**
+ * Bind the http schema CommitterFactory to {@link SimpleCommitterFactory}.
+ * @param conf config to patch
+ */
+ private Configuration httpToSimpleFactory(Configuration conf) {
+ conf.set(HTTP_COMMITTER_FACTORY, SimpleCommitterFactory.class.getName());
+ return conf;
+ }
+
+
+ /**
+ * Create a configuration with the http schema bonded to the simple factory.
+ * @return a new, patched configuration
+ */
+ private Configuration newBondedConfiguration() {
+ return httpToSimpleFactory(new Configuration());
+ }
+
+ /**
+ * Extract the (mandatory) cause of an exception.
+ * @param ex exception
+ * @param clazz expected class
+ * @return the cause, which will be of the expected type
+ * @throws AssertionError if there is a problem
+ */
+ private <E extends Throwable> E verifyCauseClass(Throwable ex,
+ Class<E> clazz) throws AssertionError {
+ Throwable cause = ex.getCause();
+ if (cause == null) {
+ throw new AssertionError("No cause", ex);
+ }
+ if (!cause.getClass().equals(clazz)) {
+ throw new AssertionError("Wrong cause class", cause);
+ }
+ return (E)cause;
+ }
+
+ @Test
+ public void testBadCommitterFactory() throws Throwable {
+ expectFactoryConstructionFailure(HTTP_COMMITTER_FACTORY);
+ }
+
+ @Test
+ public void testBoundCommitterWithSchema() throws Throwable {
+ // this verifies that a bound committer relays to the underlying committer
+ Configuration conf = newBondedConfiguration();
+ TestPathOutputCommitter.TaskContext tac
+ = new TestPathOutputCommitter.TaskContext(conf);
+ BindingPathOutputCommitter committer
+ = new BindingPathOutputCommitter(HTTP_PATH, tac);
+ intercept(IOException.class, "setupJob",
+ () -> committer.setupJob(tac));
+ }
+
+ @Test
+ public void testBoundCommitterWithDefault() throws Throwable {
+ // this verifies that a bound committer relays to the underlying committer
+ Configuration conf = newBondedConfiguration();
+ TestPathOutputCommitter.TaskContext tac
+ = new TestPathOutputCommitter.TaskContext(conf);
+ BindingPathOutputCommitter committer
+ = new BindingPathOutputCommitter(HDFS_PATH, tac);
+ assertEquals(FileOutputCommitter.class,
+ committer.getCommitter().getClass());
+ }
+
+ /**
+ * Set the specific key to a string which is not a factory class; expect
+ * a failure.
+ * @param key key to set
+ * @throws Throwable on a failure
+ */
+ @SuppressWarnings("ThrowableNotThrown")
+ protected void expectFactoryConstructionFailure(String key) throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(key, "Not a factory");
+ RuntimeException ex = intercept(RuntimeException.class,
+ () -> getCommitterFactory(HTTP_PATH, conf));
+ verifyCauseClass(
+ verifyCauseClass(ex, RuntimeException.class),
+ ClassNotFoundException.class);
+ }
+
+ /**
+ * A simple committer.
+ */
+ public static final class SimpleCommitter extends PathOutputCommitter {
+
+ private final Path outputPath;
+
+ public SimpleCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ this.outputPath = outputPath;
+ }
+
+ @Override
+ public Path getWorkPath() throws IOException {
+ return null;
+ }
+
+ /**
+ * Job setup throws an exception.
+ * @param jobContext Context of the job
+ * @throws IOException always
+ */
+ @Override
+ public void setupJob(JobContext jobContext) throws IOException {
+ throw new IOException("setupJob");
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext taskContext) throws IOException {
+
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext taskContext)
+ throws IOException {
+ return false;
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext taskContext) throws IOException {
+
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext taskContext) throws IOException {
+
+ }
+
+ @Override
+ public Path getOutputPath() {
+ return outputPath;
+ }
+ }
+
+ /**
+ * The simple committer factory.
+ */
+ private static class SimpleCommitterFactory
+ extends PathOutputCommitterFactory {
+
+ @Override
+ public PathOutputCommitter createOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new SimpleCommitter(outputPath, context);
+ }
+
+ }
+
+ /**
+ * Some other factory.
+ */
+ private static class OtherFactory extends PathOutputCommitterFactory {
+
+ /**
+ * {@inheritDoc}
+ * @param outputPath output path. This may be null.
+ * @param context context
+ * @return
+ * @throws IOException
+ */
+ @Override
+ public PathOutputCommitter createOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new SimpleCommitter(outputPath, context);
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 97ceddf..748537c 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -129,7 +129,9 @@
<!-- surefire.forkNumber won't do the parameter -->
<!-- substitution. Putting a prefix in front of it like -->
<!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <!-- Important: Those leading 0s are needed to guarantee that -->
+ <!-- trailing three chars are always numeric and unique -->
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<!-- Propagate scale parameters -->
<fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
<fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
@@ -165,7 +167,7 @@
<!-- surefire.forkNumber won't do the parameter -->
<!-- substitution. Putting a prefix in front of it like -->
<!-- "fork-" makes it work. -->
- <test.unique.fork.id>fork-${surefire.forkNumber}</test.unique.fork.id>
+ <test.unique.fork.id>fork-000${surefire.forkNumber}</test.unique.fork.id>
<!-- Propagate scale parameters -->
<fs.s3a.scale.test.enabled>${fs.s3a.scale.test.enabled}</fs.s3a.scale.test.enabled>
<fs.s3a.scale.test.huge.filesize>${fs.s3a.scale.test.huge.filesize}</fs.s3a.scale.test.huge.filesize>
@@ -192,7 +194,6 @@
<include>**/ITest*.java</include>
</includes>
<excludes>
- <exclude>**/ITestJets3tNativeS3FileSystemContract.java</exclude>
<exclude>**/ITestS3AContractRootDir.java</exclude>
<exclude>**/ITestS3AFileContextStatistics.java</exclude>
<exclude>**/ITestS3AEncryptionSSEC*.java</exclude>
@@ -225,7 +226,6 @@
<!-- Do a sequential run for tests that cannot handle -->
<!-- parallel execution. -->
<includes>
- <include>**/ITestJets3tNativeS3FileSystemContract.java</include>
<include>**/ITestS3AContractRootDir.java</include>
<include>**/ITestS3AFileContextStatistics.java</include>
<include>**/ITestS3AHuge*.java</include>
@@ -465,8 +465,8 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <scope>test</scope>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -476,12 +476,23 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-hs</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
<scope>test</scope>
<type>jar</type>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
<scope>test</scope>
</dependency>
@@ -491,5 +502,28 @@
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <!-- artifacts needed to bring up a Mini MR Yarn cluster-->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-app</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
new file mode 100644
index 0000000..482c5a1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSBadRequestException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonServiceException;
+
+/**
+ * A 400 "Bad Request" exception was received.
+ * This is the general "bad parameters, headers, whatever" failure.
+ */
+public class AWSBadRequestException extends AWSServiceIOException {
+ /**
+ * HTTP status code which signals this failure mode was triggered: {@value}.
+ */
+ public static final int STATUS_CODE = 400;
+
+ /**
+ * Instantiate.
+ * @param operation operation which triggered this
+ * @param cause the underlying cause
+ */
+ public AWSBadRequestException(String operation,
+ AmazonServiceException cause) {
+ super(operation, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
index a8c01cb..22afb01 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSClientIOException.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
+import com.amazonaws.SdkBaseException;
import com.google.common.base.Preconditions;
import java.io.IOException;
@@ -31,7 +32,7 @@ public class AWSClientIOException extends IOException {
private final String operation;
public AWSClientIOException(String operation,
- AmazonClientException cause) {
+ SdkBaseException cause) {
super(cause);
Preconditions.checkArgument(operation != null, "Null 'operation' argument");
Preconditions.checkArgument(cause != null, "Null 'cause' argument");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
new file mode 100644
index 0000000..e6a23b2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSNoResponseException.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonServiceException;
+
+/**
+ * Status code 443, no response from server. This is considered idempotent.
+ */
+public class AWSNoResponseException extends AWSServiceIOException {
+ public AWSNoResponseException(String operation,
+ AmazonServiceException cause) {
+ super(operation, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
new file mode 100644
index 0000000..bb337ee
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSRedirectException.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonServiceException;
+
+/**
+ * Request is redirected.
+ * If this gets as far as the user, it's unrecoverable
+ */
+public class AWSRedirectException extends AWSServiceIOException {
+
+ /**
+ * Instantiate.
+ * @param operation operation which triggered this
+ * @param cause the underlying cause
+ */
+ public AWSRedirectException(String operation,
+ AmazonServiceException cause) {
+ super(operation, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
new file mode 100644
index 0000000..131cea7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSServiceThrottledException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonServiceException;
+
+/**
+ * Exception raised when a service was throttled.
+ */
+public class AWSServiceThrottledException extends AWSServiceIOException {
+
+ /**
+ * HTTP status code which signals this failure mode was triggered: {@value}.
+ */
+ public static final int STATUS_CODE = 503;
+
+ /**
+ * Instantiate.
+ * @param operation operation which triggered this
+ * @param cause the underlying cause
+ */
+ public AWSServiceThrottledException(String operation,
+ AmazonServiceException cause) {
+ super(operation, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
new file mode 100644
index 0000000..83be294
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/AWSStatus500Exception.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import com.amazonaws.AmazonServiceException;
+
+/**
+ * A 500 response came back from a service.
+ * This is considered <i>probably</i> retriable, That is, we assume
+ * <ol>
+ * <li>whatever error happened in the service itself to have happened
+ * before the infrastructure committed the operation.</li>
+ * <li>Nothing else got through either.</li>
+ * </ol>
+ */
+public class AWSStatus500Exception extends AWSServiceIOException {
+ public AWSStatus500Exception(String operation,
+ AmazonServiceException cause) {
+ super(operation, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BlockingThreadPoolExecutorService.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BlockingThreadPoolExecutorService.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BlockingThreadPoolExecutorService.java
index 5b25730..f13942d 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BlockingThreadPoolExecutorService.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/BlockingThreadPoolExecutorService.java
@@ -45,7 +45,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
final class BlockingThreadPoolExecutorService
extends SemaphoredDelegatingExecutor {
- private static Logger LOG = LoggerFactory
+ private static final Logger LOG = LoggerFactory
.getLogger(BlockingThreadPoolExecutorService.class);
private static final AtomicInteger POOLNUMBER = new AtomicInteger(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index d278bdf..e6b2bdb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -35,6 +35,11 @@ public final class Constants {
private Constants() {
}
+ /**
+ * default hadoop temp dir on local system: {@value}.
+ */
+ public static final String HADOOP_TMP_DIR = "hadoop.tmp.dir";
+
/** The minimum multipart size which S3 supports. */
public static final int MULTIPART_MIN_SIZE = 5 * 1024 * 1024;
@@ -328,14 +333,6 @@ public final class Constants {
@InterfaceAudience.Private
public static final int MAX_MULTIPART_COUNT = 10000;
- /**
- * Classname of the S3A-specific output committer factory. This
- * is what must be declared when attempting to use
- */
- @InterfaceStability.Unstable
- public static final String S3A_OUTPUT_COMMITTER_FACTORY =
- "org.apache.hadoop.fs.s3a.commit.S3AOutputCommitterFactory";
-
/* Constants. */
public static final String S3_METADATA_STORE_IMPL =
"fs.s3a.metadatastore.impl";
@@ -411,13 +408,6 @@ public final class Constants {
public static final int S3GUARD_DDB_BACKGROUND_SLEEP_MSEC_DEFAULT = 25;
/**
- * V1 committer.
- */
- @InterfaceStability.Unstable
- public static final String S3A_OUTPUT_COMMITTER_MRV1 =
- "org.apache.hadoop.fs.s3a.commit.S3OutputCommitterMRv1";
-
- /**
* The default "Null" metadata store: {@value}.
*/
@InterfaceStability.Unstable
@@ -463,4 +453,56 @@ public final class Constants {
@InterfaceStability.Unstable
public static final int DEFAULT_LIST_VERSION = 2;
+ @InterfaceStability.Unstable
+ public static final String FAIL_INJECT_THROTTLE_PROBABILITY =
+ "fs.s3a.failinject.throttle.probability";
+
+ @InterfaceStability.Unstable
+ public static final String FAIL_INJECT_CLIENT_FACTORY =
+ "org.apache.hadoop.fs.s3a.InconsistentS3ClientFactory";
+
+ /**
+ * Number of times to retry any repeatable S3 client request on failure,
+ * excluding throttling requests: {@value}.
+ */
+ public static final String RETRY_LIMIT = "fs.s3a.retry.limit";
+
+ /**
+ * Default retry limit: {@value}.
+ */
+ public static final int RETRY_LIMIT_DEFAULT = DEFAULT_MAX_ERROR_RETRIES;
+
+ /**
+ * Interval between retry attempts.: {@value}.
+ */
+ public static final String RETRY_INTERVAL = "fs.s3a.retry.interval";
+
+ /**
+ * Default retry interval: {@value}.
+ */
+ public static final String RETRY_INTERVAL_DEFAULT = "500ms";
+
+ /**
+ * Number of times to retry any throttled request: {@value}.
+ */
+ public static final String RETRY_THROTTLE_LIMIT =
+ "fs.s3a.retry.throttle.limit";
+
+ /**
+ * Default throttled retry limit: {@value}.
+ */
+ public static final int RETRY_THROTTLE_LIMIT_DEFAULT =
+ DEFAULT_MAX_ERROR_RETRIES;
+
+ /**
+ * Interval between retry attempts on throttled requests: {@value}.
+ */
+ public static final String RETRY_THROTTLE_INTERVAL =
+ "fs.s3a.retry.throttle.interval";
+
+ /**
+ * Default throttled retry interval: {@value}.
+ */
+ public static final String RETRY_THROTTLE_INTERVAL_DEFAULT = "500ms";
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
index 6476f5d..d158061 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/InconsistentAmazonS3Client.java
@@ -18,38 +18,50 @@
package org.apache.hadoop.fs.s3a;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.stream.Collectors;
+
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SdkClientException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.DeleteObjectRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsResult;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.ListObjectsV2Result;
+import com.amazonaws.services.s3.model.MultipartUploadListing;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.s3a.Constants.*;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
/**
* A wrapper around {@link com.amazonaws.services.s3.AmazonS3} that injects
* inconsistency and/or errors. Used for testing S3Guard.
@@ -88,6 +100,21 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
private long delayKeyMsec;
/**
+ * Probability of throttling a request.
+ */
+ private float throttleProbability;
+
+ /**
+ * Counter of failures since last reset.
+ */
+ private final AtomicLong failureCounter = new AtomicLong(0);
+
+ /**
+ * limit for failures before operations succeed; if 0 then "no limit".
+ */
+ private int failureLimit = 0;
+
+ /**
* Composite of data we need to track about recently deleted objects:
* when it was deleted (same was with recently put objects) and the object
* summary (since we should keep returning it for sometime after its
@@ -134,12 +161,25 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
if (delayKeySubstring.equals(MATCH_ALL_KEYS)) {
delayKeySubstring = "";
}
- delayKeyProbability = conf.getFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY,
- DEFAULT_DELAY_KEY_PROBABILITY);
+ delayKeyProbability = validProbability(
+ conf.getFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY,
+ DEFAULT_DELAY_KEY_PROBABILITY));
delayKeyMsec = conf.getLong(FAIL_INJECT_INCONSISTENCY_MSEC,
DEFAULT_DELAY_KEY_MSEC);
- LOG.info("Enabled with {} msec delay, substring {}, probability {}",
- delayKeyMsec, delayKeySubstring, delayKeyProbability);
+ setThrottleProbability(conf.getFloat(FAIL_INJECT_THROTTLE_PROBABILITY,
+ 0.0f));
+ LOG.info("{}", this);
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "Inconsistent S3 Client with"
+ + " %s msec delay, substring %s, delay probability %s;"
+ + " throttle probability %s"
+ + "; failure limit %d, failure count %d",
+ delayKeyMsec, delayKeySubstring, delayKeyProbability,
+ throttleProbability, failureLimit, failureCounter.get());
}
/**
@@ -174,10 +214,11 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
public DeleteObjectsResult deleteObjects(DeleteObjectsRequest
deleteObjectsRequest)
throws AmazonClientException, AmazonServiceException {
+ maybeFail();
for (DeleteObjectsRequest.KeyVersion keyVersion :
deleteObjectsRequest.getKeys()) {
- registerDeleteObject(keyVersion.getKey(), deleteObjectsRequest
- .getBucketName());
+ registerDeleteObject(keyVersion.getKey(),
+ deleteObjectsRequest.getBucketName());
}
return super.deleteObjects(deleteObjectsRequest);
}
@@ -187,6 +228,7 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
throws AmazonClientException, AmazonServiceException {
String key = deleteObjectRequest.getKey();
LOG.debug("key {}", key);
+ maybeFail();
registerDeleteObject(key, deleteObjectRequest.getBucketName());
super.deleteObject(deleteObjectRequest);
}
@@ -196,6 +238,7 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
public PutObjectResult putObject(PutObjectRequest putObjectRequest)
throws AmazonClientException, AmazonServiceException {
LOG.debug("key {}", putObjectRequest.getKey());
+ maybeFail();
registerPutObject(putObjectRequest);
return super.putObject(putObjectRequest);
}
@@ -204,6 +247,20 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest)
throws AmazonClientException, AmazonServiceException {
+ maybeFail();
+ return innerlistObjects(listObjectsRequest);
+ }
+
+ /**
+ * Run the list object call without any failure probability.
+ * This stops a very aggressive failure rate from completely overloading
+ * the retry logic.
+ * @param listObjectsRequest request
+ * @return listing
+ * @throws AmazonClientException failure
+ */
+ private ObjectListing innerlistObjects(ListObjectsRequest listObjectsRequest)
+ throws AmazonClientException, AmazonServiceException {
LOG.debug("prefix {}", listObjectsRequest.getPrefix());
ObjectListing listing = super.listObjects(listObjectsRequest);
listing = filterListObjects(listing);
@@ -215,6 +272,16 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
@Override
public ListObjectsV2Result listObjectsV2(ListObjectsV2Request request)
throws AmazonClientException, AmazonServiceException {
+ maybeFail();
+ return innerListObjectsV2(request);
+ }
+
+ /**
+ * Non failing V2 list object request.
+ * @param request request
+ * @return result.
+ */
+ private ListObjectsV2Result innerListObjectsV2(ListObjectsV2Request request) {
LOG.debug("prefix {}", request.getPrefix());
ListObjectsV2Result listing = super.listObjectsV2(request);
listing = filterListObjectsV2(listing);
@@ -222,17 +289,13 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
return listing;
}
-
private void addSummaryIfNotPresent(List<S3ObjectSummary> list,
S3ObjectSummary item) {
// Behavior of S3ObjectSummary
String key = item.getKey();
- for (S3ObjectSummary member : list) {
- if (member.getKey().equals(key)) {
- return;
- }
+ if (list.stream().noneMatch((member) -> member.getKey().equals(key))) {
+ list.add(item);
}
- list.add(item);
}
/**
@@ -396,13 +459,9 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
}
private List<String> filterPrefixes(List<String> prefixes) {
- List<String> outputPrefixes = new ArrayList<>();
- for (String key : prefixes) {
- if (!isKeyDelayed(delayedPutKeys.get(key), key)) {
- outputPrefixes.add(key);
- }
- }
- return outputPrefixes;
+ return prefixes.stream()
+ .filter(key -> !isKeyDelayed(delayedPutKeys.get(key), key))
+ .collect(Collectors.toList());
}
private boolean isKeyDelayed(Long enqueueTime, String key) {
@@ -425,14 +484,14 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
private void registerDeleteObject(String key, String bucket) {
if (shouldDelay(key)) {
// Record summary so we can add it back for some time post-deletion
- S3ObjectSummary summary = null;
- ObjectListing list = listObjects(bucket, key);
- for (S3ObjectSummary result : list.getObjectSummaries()) {
- if (result.getKey().equals(key)) {
- summary = result;
- break;
- }
- }
+ ListObjectsRequest request = new ListObjectsRequest()
+ .withBucketName(bucket)
+ .withPrefix(key);
+ S3ObjectSummary summary = innerlistObjects(request).getObjectSummaries()
+ .stream()
+ .filter(result -> result.getKey().equals(key))
+ .findFirst()
+ .orElse(null);
delayedDeletes.put(key, new Delete(System.currentTimeMillis(), summary));
}
}
@@ -471,7 +530,109 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
delayedPutKeys.put(key, System.currentTimeMillis());
}
+ @Override
+ public CompleteMultipartUploadResult completeMultipartUpload(
+ CompleteMultipartUploadRequest completeMultipartUploadRequest)
+ throws SdkClientException, AmazonServiceException {
+ maybeFail();
+ return super.completeMultipartUpload(completeMultipartUploadRequest);
+ }
+
+ @Override
+ public UploadPartResult uploadPart(UploadPartRequest uploadPartRequest)
+ throws SdkClientException, AmazonServiceException {
+ maybeFail();
+ return super.uploadPart(uploadPartRequest);
+ }
+
+ @Override
+ public InitiateMultipartUploadResult initiateMultipartUpload(
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest)
+ throws SdkClientException, AmazonServiceException {
+ maybeFail();
+ return super.initiateMultipartUpload(initiateMultipartUploadRequest);
+ }
+
+ @Override
+ public MultipartUploadListing listMultipartUploads(
+ ListMultipartUploadsRequest listMultipartUploadsRequest)
+ throws SdkClientException, AmazonServiceException {
+ maybeFail();
+ return super.listMultipartUploads(listMultipartUploadsRequest);
+ }
+
+ public float getDelayKeyProbability() {
+ return delayKeyProbability;
+ }
+
+ public long getDelayKeyMsec() {
+ return delayKeyMsec;
+ }
+
+ /**
+ * Get the probability of the request being throttled.
+ * @return a value 0 - 1.0f.
+ */
+ public float getThrottleProbability() {
+ return throttleProbability;
+ }
+
+ /**
+ * Set the probability of throttling a request.
+ * @param throttleProbability the probability of a request being throttled.
+ */
+ public void setThrottleProbability(float throttleProbability) {
+ this.throttleProbability = validProbability(throttleProbability);
+ }
+
+ /**
+ * Validate a probability option.
+ * @param p probability
+ * @return the probability, if valid
+ * @throws IllegalArgumentException if the probability is out of range.
+ */
+ private float validProbability(float p) {
+ Preconditions.checkArgument(p >= 0.0f && p <= 1.0f,
+ "Probability out of range 0 to 1 %s", p);
+ return p;
+ }
+
+ /**
+ * Conditionally fail the operation.
+ * @throws AmazonClientException if the client chooses to fail
+ * the request.
+ */
+ private void maybeFail() throws AmazonClientException {
+ // code structure here is to line up for more failures later
+ AmazonServiceException ex = null;
+ if (trueWithProbability(throttleProbability)) {
+ // throttle the request
+ ex = new AmazonServiceException("throttled"
+ + " count = " + (failureCounter.get() + 1), null);
+ ex.setStatusCode(503);
+ }
+
+ if (ex != null) {
+ long count = failureCounter.incrementAndGet();
+ if (failureLimit == 0
+ || (failureLimit > 0 && count < failureLimit)) {
+ throw ex;
+ }
+ }
+ }
+
+ /**
+ * Set the limit on failures before all operations pass through.
+ * This resets the failure count.
+ * @param limit limit; "0" means "no limit"
+ */
+ public void setFailureLimit(int limit) {
+ this.failureLimit = limit;
+ failureCounter.set(0);
+ }
+
/** Since ObjectListing is immutable, we just override it with wrapper. */
+ @SuppressWarnings("serial")
private static class CustomObjectListing extends ObjectListing {
private final List<S3ObjectSummary> customListing;
@@ -506,6 +667,7 @@ public class InconsistentAmazonS3Client extends AmazonS3Client {
}
}
+ @SuppressWarnings("serial")
private static class CustomListObjectsV2Result extends ListObjectsV2Result {
private final List<S3ObjectSummary> customListing;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
new file mode 100644
index 0000000..9900f4c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Invoker.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.Optional;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.SdkBaseException;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.io.retry.RetryPolicy;
+
+/**
+ * Class to provide lambda expression invocation of AWS operations.
+ *
+ * The core retry logic is in
+ * {@link #retryUntranslated(String, boolean, Retried, Operation)};
+ * the other {@code retry() and retryUntranslated()} calls are wrappers.
+ *
+ * The static {@link #once(String, String, Operation)} and
+ * {@link #once(String, String, VoidOperation)} calls take an operation and
+ * return it with AWS exceptions translated to IOEs of some form.
+ *
+ * The retry logic on a failure is defined by the retry policy passed in
+ * the constructor; the standard retry policy is {@link S3ARetryPolicy},
+ * though others may be used.
+ *
+ * The constructor also takes two {@link Retried} callbacks.
+ * The {@code caughtCallback} is called whenever an exception (IOE or AWS)
+ * is caught, before the retry processing looks at it.
+ * The {@code retryCallback} is invoked after a retry is scheduled
+ * but before the sleep.
+ * These callbacks can be used for reporting and incrementing statistics.
+ *
+ * The static {@link #quietly(String, String, VoidOperation)} and
+ * {@link #quietlyEval(String, String, Operation)} calls exist to take any
+ * operation and quietly catch & log at debug. The return value of
+ * {@link #quietlyEval(String, String, Operation)} is a java 8 optional,
+ * which can then be used in java8-expressions.
+ */
+public class Invoker {
+ private static final Logger LOG = LoggerFactory.getLogger(Invoker.class);
+
+ /**
+ * Retry policy to use.
+ */
+ private final RetryPolicy retryPolicy;
+
+ /**
+ * Default retry handler.
+ */
+ private final Retried retryCallback;
+
+ /**
+ * Instantiate.
+ * @param retryPolicy retry policy for all operations.
+ * @param retryCallback standard retry policy
+ */
+ public Invoker(
+ RetryPolicy retryPolicy,
+ Retried retryCallback) {
+ this.retryPolicy = retryPolicy;
+ this.retryCallback = retryCallback;
+ }
+
+ public RetryPolicy getRetryPolicy() {
+ return retryPolicy;
+ }
+
+ public Retried getRetryCallback() {
+ return retryCallback;
+ }
+
+ /**
+ * Execute a function, translating any exception into an IOException.
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param operation operation to execute
+ * @param <T> type of return value
+ * @return the result of the function call
+ * @throws IOException any IOE raised, or translated exception
+ */
+ @Retries.OnceTranslated
+ public static <T> T once(String action, String path, Operation<T> operation)
+ throws IOException {
+ try {
+ return operation.execute();
+ } catch (AmazonClientException e) {
+ throw S3AUtils.translateException(action, path, e);
+ }
+ }
+
+ /**
+ * Execute an operation with no result.
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param operation operation to execute
+ * @throws IOException any IOE raised, or translated exception
+ */
+ @Retries.OnceTranslated
+ public static void once(String action, String path, VoidOperation operation)
+ throws IOException {
+ once(action, path,
+ () -> {
+ operation.execute();
+ return null;
+ });
+ }
+
+ /**
+ * Execute an operation and ignore all raised IOExceptions; log at INFO.
+ * @param log log to log at info.
+ * @param action action to include in log
+ * @param path optional path to include in log
+ * @param operation operation to execute
+ * @param <T> type of operation
+ */
+ public static <T> void ignoreIOExceptions(
+ Logger log,
+ String action,
+ String path,
+ Operation<T> operation) {
+ try {
+ once(action, path, operation);
+ } catch (IOException e) {
+ log.info("{}: {}", toDescription(action, path), e.toString(), e);
+ }
+ }
+
+ /**
+ * Execute an operation and ignore all raised IOExceptions; log at INFO.
+ * @param log log to log at info.
+ * @param action action to include in log
+ * @param path optional path to include in log
+ * @param operation operation to execute
+ */
+ public static void ignoreIOExceptions(
+ Logger log,
+ String action,
+ String path,
+ VoidOperation operation) {
+ ignoreIOExceptions(log, action, path,
+ () -> {
+ operation.execute();
+ return null;
+ });
+ }
+
+ /**
+ * Execute a void operation with retry processing.
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param retrying callback on retries
+ * @param operation operation to execute
+ * @throws IOException any IOE raised, or translated exception
+ */
+ @Retries.RetryTranslated
+ public void retry(String action,
+ String path,
+ boolean idempotent,
+ Retried retrying,
+ VoidOperation operation)
+ throws IOException {
+ retry(action, path, idempotent, retrying,
+ () -> {
+ operation.execute();
+ return null;
+ });
+ }
+
+ /**
+ * Execute a void operation with the default retry callback invoked.
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param operation operation to execute
+ * @throws IOException any IOE raised, or translated exception
+ */
+ @Retries.RetryTranslated
+ public void retry(String action,
+ String path,
+ boolean idempotent,
+ VoidOperation operation)
+ throws IOException {
+ retry(action, path, idempotent, retryCallback, operation);
+ }
+
+ /**
+ * Execute a function with the default retry callback invoked.
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param operation operation to execute
+ * @param <T> type of return value
+ * @return the result of the call
+ * @throws IOException any IOE raised, or translated exception
+ */
+ @Retries.RetryTranslated
+ public <T> T retry(String action,
+ String path,
+ boolean idempotent,
+ Operation<T> operation)
+ throws IOException {
+
+ return retry(action, path, idempotent, retryCallback, operation);
+ }
+
+ /**
+ * Execute a function with retry processing.
+ * Uses {@link #once(String, String, Operation)} as the inner
+ * invocation mechanism before retry logic is performed.
+ * @param <T> type of return value
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param retrying callback on retries
+ * @param operation operation to execute
+ * @return the result of the call
+ * @throws IOException any IOE raised, or translated exception
+ */
+ @Retries.RetryTranslated
+ public <T> T retry(
+ String action,
+ String path,
+ boolean idempotent,
+ Retried retrying,
+ Operation<T> operation)
+ throws IOException {
+ return retryUntranslated(
+ toDescription(action, path),
+ idempotent,
+ retrying,
+ () -> once(action, path, operation));
+ }
+
+ /**
+ * Execute a function with retry processing and no translation.
+ * and the default retry callback.
+ * @param text description for the catching callback
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param operation operation to execute
+ * @param <T> type of return value
+ * @return the result of the call
+ * @throws IOException any IOE raised
+ * @throws RuntimeException any Runtime exception raised
+ */
+ @Retries.RetryRaw
+ public <T> T retryUntranslated(
+ String text,
+ boolean idempotent,
+ Operation<T> operation) throws IOException {
+ return retryUntranslated(text, idempotent,
+ retryCallback, operation);
+ }
+
+ /**
+ * Execute a function with retry processing: AWS SDK Exceptions
+ * are <i>not</i> translated.
+ * This is method which the others eventually invoke.
+ * @param <T> type of return value
+ * @param text text to include in messages
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param retrying callback on retries
+ * @param operation operation to execute
+ * @return the result of the call
+ * @throws IOException any IOE raised
+ * @throws SdkBaseException any AWS exception raised
+ * @throws RuntimeException : these are never caught and retries.
+ */
+ @Retries.RetryRaw
+ public <T> T retryUntranslated(
+ String text,
+ boolean idempotent,
+ Retried retrying,
+ Operation<T> operation) throws IOException {
+
+ Preconditions.checkArgument(retrying != null, "null retrying argument");
+ int retryCount = 0;
+ Exception caught;
+ RetryPolicy.RetryAction retryAction;
+ boolean shouldRetry;
+ do {
+ try {
+ // execute the operation, returning if successful
+ return operation.execute();
+ } catch (IOException | SdkBaseException e) {
+ caught = e;
+ }
+ // you only get here if the operation didn't complete
+ // normally, hence caught != null
+
+ // translate the exception into an IOE for the retry logic
+ IOException translated;
+ if (caught instanceof IOException) {
+ translated = (IOException) caught;
+ } else {
+ translated = S3AUtils.translateException(text, "",
+ (SdkBaseException)caught);
+ }
+
+
+ int attempts = retryCount + 1;
+ try {
+ // decide action base on operation, invocation count, etc
+ retryAction = retryPolicy.shouldRetry(translated, retryCount, 0,
+ idempotent);
+ // is it a retry operation?
+ shouldRetry = retryAction.action.equals(
+ RetryPolicy.RetryAction.RETRY.action);
+ if (shouldRetry) {
+ // notify the callback
+ retrying.onFailure(text, translated, retryCount, idempotent);
+ // then sleep for the policy delay
+ Thread.sleep(retryAction.delayMillis);
+ }
+ // increment the retry count
+ retryCount++;
+ } catch (InterruptedException e) {
+ // sleep was interrupted
+ // change the exception
+ caught = new InterruptedIOException("Interrupted");
+ caught.initCause(e);
+ // no retry
+ shouldRetry = false;
+ // and re-interrupt the thread
+ Thread.currentThread().interrupt();
+ } catch (Exception e) {
+ // The retry policy raised an exception
+ // log that something happened
+ LOG.warn("{}: exception in retry processing", text, e);
+ // and fail the execution with the last execution exception.
+ shouldRetry = false;
+ }
+ } while (shouldRetry);
+
+ if (caught instanceof IOException) {
+ throw (IOException) caught;
+ } else {
+ throw (SdkBaseException) caught;
+ }
+ }
+
+
+ /**
+ * Execute an operation; any exception raised is simply caught and
+ * logged at debug.
+ * @param action action to execute
+ * @param path path (for exception construction)
+ * @param operation operation
+ */
+ public static void quietly(String action,
+ String path,
+ VoidOperation operation) {
+ try {
+ once(action, path, operation);
+ } catch (Exception e) {
+ LOG.debug("Action {} failed", action, e);
+ }
+ }
+
+ /**
+ * Execute an operation; any exception raised is caught and
+ * logged at debug.
+ * The result is only non-empty if the operation succeeded
+ * @param action action to execute
+ * @param path path (for exception construction)
+ * @param operation operation
+ */
+ public static <T> Optional<T> quietlyEval(String action,
+ String path,
+ Operation<T> operation) {
+ try {
+ return Optional.of(once(action, path, operation));
+ } catch (Exception e) {
+ LOG.debug("Action {} failed", action, e);
+ return Optional.empty();
+ }
+ }
+
+ /**
+ * Take an action and path and produce a string for logging.
+ * @param action action
+ * @param path path (may be null or empty)
+ * @return string for logs
+ */
+ private static String toDescription(String action, String path) {
+ return action +
+ (StringUtils.isNotEmpty(path) ? (" on " + path) : "");
+ }
+
+ /**
+ * Arbitrary operation throwing an IOException.
+ * @param <T> return type
+ */
+ @FunctionalInterface
+ public interface Operation<T> {
+ T execute() throws IOException;
+ }
+
+ /**
+ * Void operation which may raise an IOException.
+ */
+ @FunctionalInterface
+ public interface VoidOperation {
+ void execute() throws IOException;
+ }
+
+ /**
+ * Callback for retry and notification operations.
+ * Even if the interface is throwing up "raw" exceptions, this handler
+ * gets the translated one.
+ */
+ @FunctionalInterface
+ public interface Retried {
+ /**
+ * Retry event in progress (before any sleep).
+ * @param text text passed in to the retry() Call.
+ * @param exception the caught (and possibly translated) exception.
+ * @param retries number of retries so far
+ * @param idempotent is the request idempotent.
+ */
+ void onFailure(
+ String text,
+ IOException exception,
+ int retries,
+ boolean idempotent);
+ }
+
+ /**
+ * No op for a retrying callback.
+ */
+ public static final Retried NO_OP = new Retried() {
+ @Override
+ public void onFailure(String text,
+ IOException exception,
+ int retries,
+ boolean idempotent) {
+ }
+ };
+
+ /**
+ * Log summary at info, full stack at debug.
+ */
+ public static final Retried LOG_EVENT = new Retried() {
+ @Override
+ public void onFailure(String text,
+ IOException exception,
+ int retries,
+ boolean idempotent) {
+ LOG.debug("{}: " + exception, text);
+ if (retries == 1) {
+ // stack on first attempt, to keep noise down
+ LOG.debug("{}: " + exception, text, exception);
+ }
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
index d9f059b..eb87705 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java
@@ -385,8 +385,9 @@ public class Listing {
status = statusBatchIterator.next();
// We remove from provided list the file status listed by S3 so that
// this does not return duplicate items.
- LOG.debug("Removing the status from provided file status {}", status);
- providedStatus.remove(status);
+ if (providedStatus.remove(status)) {
+ LOG.debug("Removed the status from provided file status {}", status);
+ }
} else {
if (providedStatusIterator.hasNext()) {
status = providedStatusIterator.next();
@@ -540,10 +541,11 @@ public class Listing {
* initial set of results/fail if there was a problem talking to the bucket.
* @param listPath path of the listing
* @param request initial request to make
- * */
+ * @throws IOException if listObjects raises one.
+ */
ObjectListingIterator(
Path listPath,
- S3ListRequest request) {
+ S3ListRequest request) throws IOException {
this.listPath = listPath;
this.maxKeys = owner.getMaxKeys();
this.objects = owner.listObjects(request);
@@ -571,6 +573,7 @@ public class Listing {
* @throws NoSuchElementException if there is no more data to list.
*/
@Override
+ @Retries.RetryTranslated
public S3ListResult next() throws IOException {
if (firstListing) {
// on the first listing, don't request more data.
@@ -814,19 +817,4 @@ public class Listing {
}
}
- /**
- * A Path filter which accepts all filenames.
- */
- static final PathFilter ACCEPT_ALL = new PathFilter() {
- @Override
- public boolean accept(Path file) {
- return true;
- }
-
- @Override
- public String toString() {
- return "ACCEPT_ALL";
- }
- };
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Retries.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Retries.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Retries.java
new file mode 100644
index 0000000..80ecf0c
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Retries.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Declaration of retry policy for documentation only.
+ * This is purely for visibility in source and is currently package-scoped.
+ * Compare with {@link org.apache.hadoop.io.retry.AtMostOnce}
+ * and {@link org.apache.hadoop.io.retry.Idempotent}; these are real
+ * markers used by Hadoop RPC.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Retries {
+ /**
+ * No retry, exceptions are translated.
+ */
+ @Documented
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface OnceTranslated {
+ String value() default "";
+ }
+
+ /**
+ * No retry, exceptions are not translated.
+ */
+ @Documented
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface OnceRaw {
+ String value() default "";
+ }
+
+ /**
+ * No retry, expect a bit of both.
+ */
+ @Documented
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface OnceMixed {
+ String value() default "";
+ }
+
+ /**
+ * Retried, exceptions are translated.
+ */
+ @Documented
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface RetryTranslated {
+ String value() default "";
+ }
+
+ /**
+ * Retried, no translation.
+ */
+ @Documented
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface RetryRaw {
+ String value() default "";
+ }
+
+ /**
+ * Retried, mixed translation.
+ */
+ @Documented
+ @Retention(RetentionPolicy.SOURCE)
+ public @interface RetryMixed {
+ String value() default "";
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/49] hadoop git commit: HADOOP-15067. GC time percentage reported
in JvmMetrics should be a gauge, not counter. Contributed by Misha Dmitriev.
Posted by su...@apache.org.
HADOOP-15067. GC time percentage reported in JvmMetrics should be a gauge, not counter. Contributed by Misha Dmitriev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d162252d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d162252d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d162252d
Branch: refs/heads/YARN-5881
Commit: d162252d7a7223631ff66ba0210953296407e55f
Parents: b46ca7e
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Nov 23 09:00:59 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Nov 23 09:01:28 2017 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java | 2 +-
.../java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d162252d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
index 8c3375f..5f9afdd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/source/JvmMetrics.java
@@ -188,7 +188,7 @@ public class JvmMetrics implements MetricsSource {
}
if (gcTimeMonitor != null) {
- rb.addCounter(GcTimePercentage,
+ rb.addGauge(GcTimePercentage,
gcTimeMonitor.getLatestGcData().getGcTimePercentage());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d162252d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
index 5320b6e..aa1b009 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/metrics2/source/TestJvmMetrics.java
@@ -101,7 +101,7 @@ public class TestJvmMetrics {
verify(rb).tag(SessionId, "test");
for (JvmMetricsInfo info : JvmMetricsInfo.values()) {
if (info.name().equals("GcTimePercentage")) {
- verify(rb).addCounter(eq(info), anyInt());
+ verify(rb).addGauge(eq(info), anyInt());
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitterFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitterFactory.java
new file mode 100644
index 0000000..b446f22
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitterFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitterFactory;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+
+/**
+ * Factory for the {@link PartitionedStagingCommitter}.
+ */
+public class PartitionedStagingCommitterFactory
+ extends AbstractS3ACommitterFactory {
+
+ /**
+ * Name of this class: {@value}.
+ */
+ public static final String CLASSNAME
+ = "org.apache.hadoop.fs.s3a.commit.staging"
+ + ".PartitionedStagingCommitterFactory";
+
+ public PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem,
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new PartitionedStagingCommitter(outputPath, context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
new file mode 100644
index 0000000..a4d39d7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/Paths.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIsDirectoryException;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants.*;
+
+/**
+ * Path operations for the staging committers.
+ */
+public final class Paths {
+
+ private Paths() {
+ }
+
+ /**
+ * Insert the UUID to a path if it is not there already.
+ * If there is a trailing "." in the prefix after the last slash, the
+ * UUID is inserted before it with a "-" prefix; otherwise appended.
+ *
+ * Examples:
+ * <pre>
+ * /example/part-0000 ==> /example/part-0000-0ab34
+ * /example/part-0001.gz.csv ==> /example/part-0001-0ab34.gz.csv
+ * /example/part-0002-0abc3.gz.csv ==> /example/part-0002-0abc3.gz.csv
+ * /example0abc3/part-0002.gz.csv ==> /example0abc3/part-0002.gz.csv
+ * </pre>
+ *
+ *
+ * @param pathStr path as a string; must not have a trailing "/".
+ * @param uuid UUID to append; must not be empty
+ * @return new path.
+ */
+ public static String addUUID(String pathStr, String uuid) {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(pathStr), "empty path");
+ Preconditions.checkArgument(StringUtils.isNotEmpty(uuid), "empty uuid");
+ // In some cases, Spark will add the UUID to the filename itself.
+ if (pathStr.contains(uuid)) {
+ return pathStr;
+ }
+
+ int dot; // location of the first '.' in the file name
+ int lastSlash = pathStr.lastIndexOf('/');
+ if (lastSlash >= 0) {
+ Preconditions.checkState(lastSlash + 1 < pathStr.length(),
+ "Bad path: " + pathStr);
+ dot = pathStr.indexOf('.', lastSlash);
+ } else {
+ dot = pathStr.indexOf('.');
+ }
+
+ if (dot >= 0) {
+ return pathStr.substring(0, dot) + "-" + uuid + pathStr.substring(dot);
+ } else {
+ return pathStr + "-" + uuid;
+ }
+ }
+
+ /**
+ * Get the parent path of a string path: everything up to but excluding
+ * the last "/" in the path.
+ * @param pathStr path as a string
+ * @return the parent or null if there is no parent.
+ */
+ public static String getParent(String pathStr) {
+ int lastSlash = pathStr.lastIndexOf('/');
+ if (lastSlash >= 0) {
+ return pathStr.substring(0, lastSlash);
+ }
+ return null;
+ }
+
+ /**
+ * Using {@code URI#relativize()}, build the relative path from the
+ * base path to the full path.
+ * If {@code childPath} is not a child of {@code basePath} the outcome
+ * os undefined.
+ * @param basePath base path
+ * @param fullPath full path under the base path.
+ * @return the relative path
+ */
+ public static String getRelativePath(Path basePath,
+ Path fullPath) {
+ return basePath.toUri().relativize(fullPath.toUri()).getPath();
+ }
+
+ /**
+ * Varags constructor of paths. Not very efficient.
+ * @param parent parent path
+ * @param child child entries. "" elements are skipped.
+ * @return the full child path.
+ */
+ public static Path path(Path parent, String... child) {
+ Path p = parent;
+ for (String c : child) {
+ if (!c.isEmpty()) {
+ p = new Path(p, c);
+ }
+ }
+ return p;
+ }
+
+ /**
+ * A cache of temporary folders. There's a risk here that the cache
+ * gets too big
+ */
+ private static Cache<TaskAttemptID, Path> tempFolders = CacheBuilder
+ .newBuilder().build();
+
+ /**
+ * Get the task attempt temporary directory in the local filesystem.
+ * @param conf configuration
+ * @param uuid some UUID, such as a job UUID
+ * @param attemptID attempt ID
+ * @return a local task attempt directory.
+ * @throws IOException IO problem.
+ */
+ public static Path getLocalTaskAttemptTempDir(final Configuration conf,
+ final String uuid,
+ final TaskAttemptID attemptID)
+ throws IOException {
+ try {
+ final LocalDirAllocator allocator =
+ new LocalDirAllocator(Constants.BUFFER_DIR);
+ return tempFolders.get(attemptID,
+ () -> {
+ return FileSystem.getLocal(conf).makeQualified(
+ allocator.getLocalPathForWrite(uuid, conf));
+ });
+ } catch (ExecutionException e) {
+ throw new RuntimeException(e.getCause());
+ } catch (UncheckedExecutionException e) {
+ if (e.getCause() instanceof RuntimeException) {
+ throw (RuntimeException) e.getCause();
+ }
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Remove all information held about task attempts.
+ * @param attemptID attempt ID.
+ */
+ public static void clearTempFolderInfo(final TaskAttemptID attemptID) {
+ tempFolders.invalidate(attemptID);
+ }
+
+ /**
+ * Reset the temp folder cache; useful in tests.
+ */
+ @VisibleForTesting
+ public static void resetTempFolderCache() {
+ tempFolders.invalidateAll();
+ }
+
+ /**
+ * Try to come up with a good temp directory for different filesystems.
+ * @param fs filesystem
+ * @param conf configuration
+ * @return a qualified path under which temporary work can go.
+ */
+ public static Path tempDirForStaging(FileSystem fs,
+ Configuration conf) {
+ String fallbackPath = fs.getScheme().equals("file")
+ ? System.getProperty(JAVA_IO_TMPDIR)
+ : FILESYSTEM_TEMP_PATH;
+
+ return fs.makeQualified(new Path(conf.getTrimmed(
+ FS_S3A_COMMITTER_STAGING_TMP_PATH, fallbackPath)));
+ }
+
+ /**
+ * Get the Application Attempt ID for this job.
+ * @param conf the config to look in
+ * @return the Application Attempt ID for a given job.
+ */
+ private static int getAppAttemptId(Configuration conf) {
+ return conf.getInt(
+ MRJobConfig.APPLICATION_ATTEMPT_ID, 0);
+ }
+
+ /**
+ * Build a qualified temporary path for the multipart upload commit
+ * information in the cluster filesystem.
+ * Path is built by
+ * {@link #getMultipartUploadCommitsDirectory(FileSystem, Configuration, String)}
+ * @param conf configuration defining default FS.
+ * @param uuid uuid of job
+ * @return a path which can be used for temporary work
+ * @throws IOException on an IO failure.
+ */
+ public static Path getMultipartUploadCommitsDirectory(Configuration conf,
+ String uuid) throws IOException {
+ return getMultipartUploadCommitsDirectory(FileSystem.get(conf), conf, uuid);
+ }
+
+ /**
+ * Build a qualified temporary path for the multipart upload commit
+ * information in the supplied filesystem
+ * (which is expected to be the cluster FS).
+ * Currently {code $tempDir/$user/$uuid/staging-uploads} where
+ * {@code tempDir} is from
+ * {@link #tempDirForStaging(FileSystem, Configuration)}.
+ * @param fs target FS
+ * @param conf configuration
+ * @param uuid uuid of job
+ * @return a path which can be used for temporary work
+ * @throws IOException on an IO failure.
+ */
+ @VisibleForTesting
+ static Path getMultipartUploadCommitsDirectory(FileSystem fs,
+ Configuration conf, String uuid) throws IOException {
+ return path(
+ tempDirForStaging(fs, conf),
+ UserGroupInformation.getCurrentUser().getShortUserName(),
+ uuid,
+ STAGING_UPLOADS);
+ }
+
+ /**
+ * Returns the partition of a relative file path, or null if the path is a
+ * file name with no relative directory.
+ *
+ * @param relative a relative file path
+ * @return the partition of the relative file path
+ */
+ protected static String getPartition(String relative) {
+ return getParent(relative);
+ }
+
+ /**
+ * Get the set of partitions from the list of files being staged.
+ * This is all immediate parents of those files. If a file is in the root
+ * dir, the partition is declared to be
+ * {@link StagingCommitterConstants#TABLE_ROOT}.
+ * @param attemptPath path for the attempt
+ * @param taskOutput list of output files.
+ * @return list of partitions.
+ * @throws IOException IO failure
+ */
+ public static Set<String> getPartitions(Path attemptPath,
+ List<? extends FileStatus> taskOutput)
+ throws IOException {
+ // get a list of partition directories
+ Set<String> partitions = Sets.newLinkedHashSet();
+ for (FileStatus fileStatus : taskOutput) {
+ // sanity check the output paths
+ Path outputFile = fileStatus.getPath();
+ if (!fileStatus.isFile()) {
+ throw new PathIsDirectoryException(outputFile.toString());
+ }
+ String partition = getPartition(
+ getRelativePath(attemptPath, outputFile));
+ partitions.add(partition != null ? partition : TABLE_ROOT);
+ }
+
+ return partitions;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java
new file mode 100644
index 0000000..922d1ad
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitter.java
@@ -0,0 +1,851 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.DurationInfo;
+import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.fs.s3a.commit.Tasks;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
+
+import static com.google.common.base.Preconditions.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.Invoker.*;
+import static org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*;
+
+/**
+ * Committer based on the contributed work of the
+ * <a href="https://github.com/rdblue/s3committer">Netflix multipart committers.</a>
+ * <ol>
+ * <li>
+ * The working directory of each task is actually under a temporary
+ * path in the local filesystem; jobs write directly into it.
+ * </li>
+ * <li>
+ * Task Commit: list all files under the task working dir, upload
+ * each of them but do not commit the final operation.
+ * Persist the information for each pending commit into the cluster
+ * for enumeration and commit by the job committer.
+ * </li>
+ * <li>Task Abort: recursive delete of task working dir.</li>
+ * <li>Job Commit: list all pending PUTs to commit; commit them.</li>
+ * <li>
+ * Job Abort: list all pending PUTs to commit; abort them.
+ * Delete all task attempt directories.
+ * </li>
+ * </ol>
+ *
+ * This is the base class of the Partitioned and Directory committers.
+ * It does not do any conflict resolution, and is made non-abstract
+ * primarily for test purposes. It is not expected to be used in production.
+ */
+public class StagingCommitter extends AbstractS3ACommitter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ StagingCommitter.class);
+
+ /** Name: {@value}. */
+ public static final String NAME = "staging";
+ private final Path constructorOutputPath;
+ private final long uploadPartSize;
+ private final String uuid;
+ private final boolean uniqueFilenames;
+ private final FileOutputCommitter wrappedCommitter;
+
+ private ConflictResolution conflictResolution;
+ private String s3KeyPrefix;
+
+ /** The directory in the cluster FS for commits to go to. */
+ private Path commitsDirectory;
+
+ /**
+ * Committer for a single task attempt.
+ * @param outputPath final output path
+ * @param context task context
+ * @throws IOException on a failure
+ */
+ public StagingCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ this.constructorOutputPath = checkNotNull(getOutputPath(), "output path");
+ Configuration conf = getConf();
+ this.uploadPartSize = conf.getLongBytes(
+ MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
+ this.uuid = getUploadUUID(conf, context.getJobID());
+ this.uniqueFilenames = conf.getBoolean(
+ FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES,
+ DEFAULT_STAGING_COMMITTER_UNIQUE_FILENAMES);
+ setWorkPath(buildWorkPath(context, uuid));
+ this.wrappedCommitter = createWrappedCommitter(context, conf);
+ setOutputPath(constructorOutputPath);
+ Path finalOutputPath = getOutputPath();
+ Preconditions.checkNotNull(finalOutputPath, "Output path cannot be null");
+ S3AFileSystem fs = getS3AFileSystem(finalOutputPath,
+ context.getConfiguration(), false);
+ s3KeyPrefix = fs.pathToKey(finalOutputPath);
+ LOG.debug("{}: final output path is {}", getRole(), finalOutputPath);
+ // forces evaluation and caching of the resolution mode.
+ ConflictResolution mode = getConflictResolutionMode(getJobContext(),
+ fs.getConf());
+ LOG.debug("Conflict resolution mode: {}", mode);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ /**
+ * Create the wrapped committer.
+ * This includes customizing its options, and setting up the destination
+ * directory.
+ * @param context job/task context.
+ * @param conf config
+ * @return the inner committer
+ * @throws IOException on a failure
+ */
+ protected FileOutputCommitter createWrappedCommitter(JobContext context,
+ Configuration conf) throws IOException {
+
+ // explicitly choose commit algorithm
+ initFileOutputCommitterOptions(context);
+ commitsDirectory = Paths.getMultipartUploadCommitsDirectory(conf, uuid);
+ return new FileOutputCommitter(commitsDirectory, context);
+ }
+
+ /**
+ * Init the context config with everything needed for the file output
+ * committer. In particular, this code currently only works with
+ * commit algorithm 1.
+ * @param context context to configure.
+ */
+ protected void initFileOutputCommitterOptions(JobContext context) {
+ context.getConfiguration()
+ .setInt(FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, 1);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("StagingCommitter{");
+ sb.append(super.toString());
+ sb.append(", conflictResolution=").append(conflictResolution);
+ if (wrappedCommitter != null) {
+ sb.append(", wrappedCommitter=").append(wrappedCommitter);
+ }
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Get the UUID of an upload; may be the job ID.
+ * Spark will use a fake app ID based on the current minute and job ID 0.
+ * To avoid collisions, the key policy is:
+ * <ol>
+ * <li>Value of {@link InternalCommitterConstants#FS_S3A_COMMITTER_STAGING_UUID}.</li>
+ * <li>Value of {@code "spark.sql.sources.writeJobUUID"}.</li>
+ * <li>Value of {@code "spark.app.id"}.</li>
+ * <li>JobId passed in.</li>
+ * </ol>
+ * The staging UUID is set in in {@link #setupJob(JobContext)} and so will
+ * be valid in all sequences where the job has been set up for the
+ * configuration passed in.
+ * @param conf job/task configuration
+ * @param jobId Job ID
+ * @return an ID for use in paths.
+ */
+ public static String getUploadUUID(Configuration conf, String jobId) {
+ return conf.getTrimmed(
+ InternalCommitterConstants.FS_S3A_COMMITTER_STAGING_UUID,
+ conf.getTrimmed(SPARK_WRITE_UUID,
+ conf.getTrimmed(SPARK_APP_ID, jobId)));
+ }
+
+ /**
+ * Get the UUID of a Job.
+ * @param conf job/task configuration
+ * @param jobId Job ID
+ * @return an ID for use in paths.
+ */
+ public static String getUploadUUID(Configuration conf, JobID jobId) {
+ return getUploadUUID(conf, jobId.toString());
+ }
+
+ /**
+ * Get the work path for a task.
+ * @param context job/task complex
+ * @param uuid UUID
+ * @return a path or null if the context is not of a task
+ * @throws IOException failure to build the path
+ */
+ private static Path buildWorkPath(JobContext context, String uuid)
+ throws IOException {
+ if (context instanceof TaskAttemptContext) {
+ return taskAttemptWorkingPath((TaskAttemptContext) context, uuid);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Is this committer using unique filenames?
+ * @return true if unique filenames are used.
+ */
+ public Boolean useUniqueFilenames() {
+ return uniqueFilenames;
+ }
+
+ /**
+ * Get the filesystem for the job attempt.
+ * @param context the context of the job. This is used to get the
+ * application attempt ID.
+ * @return the FS to store job attempt data.
+ * @throws IOException failure to create the FS.
+ */
+ public FileSystem getJobAttemptFileSystem(JobContext context)
+ throws IOException {
+ Path p = getJobAttemptPath(context);
+ return p.getFileSystem(context.getConfiguration());
+ }
+
+ /**
+ * Compute the path where the output of a given job attempt will be placed.
+ * @param context the context of the job. This is used to get the
+ * application attempt ID.
+ * @param out the output path to place these in.
+ * @return the path to store job attempt data.
+ */
+ public static Path getJobAttemptPath(JobContext context, Path out) {
+ return getJobAttemptPath(getAppAttemptId(context), out);
+ }
+
+ /**
+ * Compute the path where the output of a given job attempt will be placed.
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @return the path to store job attempt data.
+ */
+ private static Path getJobAttemptPath(int appAttemptId, Path out) {
+ return new Path(getPendingJobAttemptsPath(out),
+ String.valueOf(appAttemptId));
+ }
+
+ @Override
+ protected Path getJobAttemptPath(int appAttemptId) {
+ return new Path(getPendingJobAttemptsPath(commitsDirectory),
+ String.valueOf(appAttemptId));
+ }
+
+ /**
+ * Compute the path where the output of pending task attempts are stored.
+ * @param context the context of the job with pending tasks.
+ * @return the path where the output of pending task attempts are stored.
+ */
+ private static Path getPendingTaskAttemptsPath(JobContext context, Path out) {
+ return new Path(getJobAttemptPath(context, out), TEMPORARY);
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until
+ * that task is committed.
+ *
+ * @param context the context of the task attempt.
+ * @param out The output path to put things in.
+ * @return the path where a task attempt should be stored.
+ */
+ public static Path getTaskAttemptPath(TaskAttemptContext context, Path out) {
+ return new Path(getPendingTaskAttemptsPath(context, out),
+ String.valueOf(context.getTaskAttemptID()));
+ }
+
+ /**
+ * Get the location of pending job attempts.
+ * @param out the base output directory.
+ * @return the location of pending job attempts.
+ */
+ private static Path getPendingJobAttemptsPath(Path out) {
+ Preconditions.checkNotNull(out, "Null 'out' path");
+ return new Path(out, TEMPORARY);
+ }
+
+ /**
+ * Compute the path where the output of a committed task is stored until
+ * the entire job is committed.
+ * @param context the context of the task attempt
+ * @return the path where the output of a committed task is stored until
+ * the entire job is committed.
+ */
+ public Path getCommittedTaskPath(TaskAttemptContext context) {
+ return getCommittedTaskPath(getAppAttemptId(context), context);
+ }
+
+ /**
+ * Validate the task attempt context; makes sure
+ * that the task attempt ID data is valid.
+ * @param context task context
+ */
+ private static void validateContext(TaskAttemptContext context) {
+ Preconditions.checkNotNull(context, "null context");
+ Preconditions.checkNotNull(context.getTaskAttemptID(),
+ "null task attempt ID");
+ Preconditions.checkNotNull(context.getTaskAttemptID().getTaskID(),
+ "null task ID");
+ Preconditions.checkNotNull(context.getTaskAttemptID().getJobID(),
+ "null job ID");
+ }
+
+ /**
+ * Compute the path where the output of a committed task is stored until the
+ * entire job is committed for a specific application attempt.
+ * @param appAttemptId the ID of the application attempt to use
+ * @param context the context of any task.
+ * @return the path where the output of a committed task is stored.
+ */
+ protected Path getCommittedTaskPath(int appAttemptId,
+ TaskAttemptContext context) {
+ validateContext(context);
+ return new Path(getJobAttemptPath(appAttemptId),
+ String.valueOf(context.getTaskAttemptID().getTaskID()));
+ }
+
+ @Override
+ public Path getTempTaskAttemptPath(TaskAttemptContext context) {
+ throw new UnsupportedOperationException("Unimplemented");
+ }
+
+ /**
+ * Lists the output of a task under the task attempt path. Subclasses can
+ * override this method to change how output files are identified.
+ * <p>
+ * This implementation lists the files that are direct children of the output
+ * path and filters hidden files (file names starting with '.' or '_').
+ * <p>
+ * The task attempt path is provided by
+ * {@link #getTaskAttemptPath(TaskAttemptContext)}
+ *
+ * @param context this task's {@link TaskAttemptContext}
+ * @return the output files produced by this task in the task attempt path
+ * @throws IOException on a failure
+ */
+ protected List<LocatedFileStatus> getTaskOutput(TaskAttemptContext context)
+ throws IOException {
+
+ // get files on the local FS in the attempt path
+ Path attemptPath = getTaskAttemptPath(context);
+ Preconditions.checkNotNull(attemptPath,
+ "No attemptPath path in {}", this);
+
+ LOG.debug("Scanning {} for files to commit", attemptPath);
+
+ return listAndFilter(getTaskAttemptFilesystem(context),
+ attemptPath, true, HIDDEN_FILE_FILTER);
+ }
+
+ /**
+ * Returns the final S3 key for a relative path. Subclasses can override this
+ * method to upload files to a different S3 location.
+ * <p>
+ * This implementation concatenates the relative path with the key prefix
+ * from the output path.
+ * If {@link CommitConstants#FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES} is
+ * set, then the task UUID is also included in the calculation
+ *
+ * @param relative the path of a file relative to the task attempt path
+ * @param context the JobContext or TaskAttemptContext for this job
+ * @return the S3 key where the file will be uploaded
+ */
+ protected String getFinalKey(String relative, JobContext context) {
+ if (uniqueFilenames) {
+ return getS3KeyPrefix(context) + "/" + Paths.addUUID(relative, uuid);
+ } else {
+ return getS3KeyPrefix(context) + "/" + relative;
+ }
+ }
+
+ /**
+ * Returns the final S3 location for a relative path as a Hadoop {@link Path}.
+ * This is a final method that calls {@link #getFinalKey(String, JobContext)}
+ * to determine the final location.
+ *
+ * @param relative the path of a file relative to the task attempt path
+ * @param context the JobContext or TaskAttemptContext for this job
+ * @return the S3 Path where the file will be uploaded
+ */
+ protected final Path getFinalPath(String relative, JobContext context)
+ throws IOException {
+ return getDestS3AFS().keyToQualifiedPath(getFinalKey(relative, context));
+ }
+
+ /**
+ * Return the local work path as the destination for writing work.
+ * @param context the context of the task attempt.
+ * @return a path in the local filesystem.
+ */
+ @Override
+ public Path getBaseTaskAttemptPath(TaskAttemptContext context) {
+ // a path on the local FS for files that will be uploaded
+ return getWorkPath();
+ }
+
+ /**
+ * For a job attempt path, the staging committer returns that of the
+ * wrapped committer.
+ * @param context the context of the job.
+ * @return a path in HDFS.
+ */
+ @Override
+ public Path getJobAttemptPath(JobContext context) {
+ return wrappedCommitter.getJobAttemptPath(context);
+ }
+
+ /**
+ * Set up the job, including calling the same method on the
+ * wrapped committer.
+ * @param context job context
+ * @throws IOException IO failure.
+ */
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ LOG.debug("{}, Setting up job {}", getRole(), jobIdString(context));
+ context.getConfiguration().set(
+ InternalCommitterConstants.FS_S3A_COMMITTER_STAGING_UUID, uuid);
+ wrappedCommitter.setupJob(context);
+ }
+
+ /**
+ * Get the list of pending uploads for this job attempt.
+ * @param context job context
+ * @return a list of pending uploads.
+ * @throws IOException Any IO failure
+ */
+ @Override
+ protected List<SinglePendingCommit> listPendingUploadsToCommit(
+ JobContext context)
+ throws IOException {
+ return listPendingUploads(context, false);
+ }
+
+ /**
+ * Get the list of pending uploads for this job attempt, swallowing
+ * exceptions.
+ * @param context job context
+ * @return a list of pending uploads. If an exception was swallowed,
+ * then this may not match the actual set of pending operations
+ * @throws IOException shouldn't be raised, but retained for the compiler
+ */
+ protected List<SinglePendingCommit> listPendingUploadsToAbort(
+ JobContext context) throws IOException {
+ return listPendingUploads(context, true);
+ }
+
+ /**
+ * Get the list of pending uploads for this job attempt.
+ * @param context job context
+ * @param suppressExceptions should exceptions be swallowed?
+ * @return a list of pending uploads. If exceptions are being swallowed,
+ * then this may not match the actual set of pending operations
+ * @throws IOException Any IO failure which wasn't swallowed.
+ */
+ protected List<SinglePendingCommit> listPendingUploads(
+ JobContext context, boolean suppressExceptions) throws IOException {
+ try {
+ Path wrappedJobAttemptPath = wrappedCommitter.getJobAttemptPath(context);
+ final FileSystem attemptFS = wrappedJobAttemptPath.getFileSystem(
+ context.getConfiguration());
+ return loadPendingsetFiles(context, suppressExceptions, attemptFS,
+ listAndFilter(attemptFS,
+ wrappedJobAttemptPath, false,
+ HIDDEN_FILE_FILTER));
+ } catch (IOException e) {
+ // unable to work with endpoint, if suppressing errors decide our actions
+ maybeIgnore(suppressExceptions, "Listing pending uploads", e);
+ }
+ // reached iff an IOE was caught and swallowed
+ return new ArrayList<>(0);
+ }
+
+ @Override
+ public void cleanupStagingDirs() {
+ Path workPath = getWorkPath();
+ if (workPath != null) {
+ LOG.debug("Cleaning up work path {}", workPath);
+ ignoreIOExceptions(LOG, "cleaning up", workPath.toString(),
+ () -> deleteQuietly(workPath.getFileSystem(getConf()),
+ workPath, true));
+ }
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ protected void cleanup(JobContext context,
+ boolean suppressExceptions)
+ throws IOException {
+ maybeIgnore(suppressExceptions, "Cleanup wrapped committer",
+ () -> wrappedCommitter.cleanupJob(context));
+ maybeIgnore(suppressExceptions, "Delete destination paths",
+ () -> deleteDestinationPaths(context));
+ super.cleanup(context, suppressExceptions);
+ }
+
+ @Override
+ protected void abortPendingUploadsInCleanup(boolean suppressExceptions)
+ throws IOException {
+ if (getConf()
+ .getBoolean(FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS, true)) {
+ super.abortPendingUploadsInCleanup(suppressExceptions);
+ } else {
+ LOG.info("Not cleanup up pending uploads to {} as {} is false ",
+ getOutputPath(),
+ FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS);
+ }
+ }
+
+ @Override
+ protected void abortJobInternal(JobContext context,
+ boolean suppressExceptions) throws IOException {
+ String r = getRole();
+ boolean failed = false;
+ try (DurationInfo d = new DurationInfo(LOG,
+ "%s: aborting job in state %s ", r, jobIdString(context))) {
+ List<SinglePendingCommit> pending = listPendingUploadsToAbort(context);
+ abortPendingUploads(context, pending, suppressExceptions);
+ } catch (FileNotFoundException e) {
+ // nothing to list
+ LOG.debug("No job directory to read uploads from");
+ } catch (IOException e) {
+ failed = true;
+ maybeIgnore(suppressExceptions, "aborting job", e);
+ } finally {
+ super.abortJobInternal(context, failed || suppressExceptions);
+ }
+ }
+
+ /**
+ * Delete the working paths of a job. Does not attempt to clean up
+ * the work of the wrapped committer.
+ * <ol>
+ * <li>The job attempt path</li>
+ * <li>$dest/__temporary</li>
+ * <li>the local working directory for staged files</li>
+ * </ol>
+ * @param context job context
+ * @throws IOException IO failure
+ */
+ protected void deleteDestinationPaths(JobContext context) throws IOException {
+ Path attemptPath = getJobAttemptPath(context);
+ ignoreIOExceptions(LOG,
+ "Deleting Job attempt Path", attemptPath.toString(),
+ () -> deleteWithWarning(
+ getJobAttemptFileSystem(context),
+ attemptPath,
+ true));
+
+ // delete the __temporary directory. This will cause problems
+ // if there is >1 task targeting the same dest dir
+ deleteWithWarning(getDestFS(),
+ new Path(getOutputPath(), TEMPORARY),
+ true);
+ // and the working path
+ deleteTaskWorkingPathQuietly(context);
+ }
+
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ Path taskAttemptPath = getTaskAttemptPath(context);
+ try (DurationInfo d = new DurationInfo(LOG,
+ "%s: setup task attempt path %s ", getRole(), taskAttemptPath)) {
+ // create the local FS
+ taskAttemptPath.getFileSystem(getConf()).mkdirs(taskAttemptPath);
+ wrappedCommitter.setupTask(context);
+ }
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "%s: needsTaskCommit() Task %s",
+ getRole(), context.getTaskAttemptID())) {
+ // check for files on the local FS in the attempt path
+ Path attemptPath = getTaskAttemptPath(context);
+ FileSystem fs = getTaskAttemptFilesystem(context);
+
+ // This could be made more efficient with a probe "hasChildren(Path)"
+ // which returns true if there is >1 entry under a given path.
+ FileStatus[] stats = fs.listStatus(attemptPath);
+ LOG.debug("{} files to commit under {}", stats.length, attemptPath);
+ return stats.length > 0;
+ } catch (FileNotFoundException e) {
+ // list didn't find a directory, so nothing to commit
+ // TODO: throw this up as an error?
+ LOG.info("No files to commit");
+ throw e;
+ }
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "%s: commit task %s", getRole(), context.getTaskAttemptID())) {
+ int count = commitTaskInternal(context, getTaskOutput(context));
+ LOG.info("{}: upload file count: {}", getRole(), count);
+ } catch (IOException e) {
+ LOG.error("{}: commit of task {} failed",
+ getRole(), context.getTaskAttemptID(), e);
+ getCommitOperations().taskCompleted(false);
+ throw e;
+ }
+ getCommitOperations().taskCompleted(true);
+ }
+
+ /**
+ * Commit the task by uploading all created files and then
+ * writing a pending entry for them.
+ * @param context task context
+ * @param taskOutput list of files from the output
+ * @return number of uploads committed.
+ * @throws IOException IO Failures.
+ */
+ protected int commitTaskInternal(final TaskAttemptContext context,
+ List<? extends FileStatus> taskOutput)
+ throws IOException {
+ LOG.debug("{}: commitTaskInternal", getRole());
+ Configuration conf = context.getConfiguration();
+
+ final Path attemptPath = getTaskAttemptPath(context);
+ FileSystem attemptFS = getTaskAttemptFilesystem(context);
+ LOG.debug("{}: attempt path is {}", getRole(), attemptPath);
+
+ // add the commits file to the wrapped committer's task attempt location.
+ // of this method.
+ Path commitsAttemptPath = wrappedCommitter.getTaskAttemptPath(context);
+ FileSystem commitsFS = commitsAttemptPath.getFileSystem(conf);
+
+ // keep track of unfinished commits in case one fails. if something fails,
+ // we will try to abort the ones that had already succeeded.
+ int commitCount = taskOutput.size();
+ final Queue<SinglePendingCommit> commits = new ConcurrentLinkedQueue<>();
+ LOG.info("{}: uploading from staging directory to S3", getRole());
+ LOG.info("{}: Saving pending data information to {}",
+ getRole(), commitsAttemptPath);
+ if (taskOutput.isEmpty()) {
+ // there is nothing to write. needsTaskCommit() should have caught
+ // this, so warn that there is some kind of problem in the protocol.
+ LOG.warn("{}: No files to commit", getRole());
+ } else {
+ boolean threw = true;
+ // before the uploads, report some progress
+ context.progress();
+
+ PendingSet pendingCommits = new PendingSet(commitCount);
+ try {
+ Tasks.foreach(taskOutput)
+ .stopOnFailure()
+ .executeWith(buildThreadPool(context))
+ .run(stat -> {
+ Path path = stat.getPath();
+ File localFile = new File(path.toUri().getPath());
+ String relative = Paths.getRelativePath(attemptPath, path);
+ String partition = Paths.getPartition(relative);
+ String key = getFinalKey(relative, context);
+ Path destPath = getDestS3AFS().keyToQualifiedPath(key);
+ SinglePendingCommit commit = getCommitOperations()
+ .uploadFileToPendingCommit(
+ localFile,
+ destPath,
+ partition,
+ uploadPartSize);
+ LOG.debug("{}: adding pending commit {}", getRole(), commit);
+ commits.add(commit);
+ });
+
+ for (SinglePendingCommit commit : commits) {
+ pendingCommits.add(commit);
+ }
+
+ // save the data
+ // although overwrite=false, there's still a risk of > 1 entry being
+ // committed if the FS doesn't have create-no-overwrite consistency.
+
+ LOG.debug("Saving {} pending commit(s)) to file {}",
+ pendingCommits.size(),
+ commitsAttemptPath);
+ pendingCommits.save(commitsFS, commitsAttemptPath, false);
+ threw = false;
+
+ } finally {
+ if (threw) {
+ LOG.error(
+ "{}: Exception during commit process, aborting {} commit(s)",
+ getRole(), commits.size());
+ Tasks.foreach(commits)
+ .suppressExceptions()
+ .run(commit -> getCommitOperations().abortSingleCommit(commit));
+ deleteTaskAttemptPathQuietly(context);
+ }
+ }
+ // always purge attempt information at this point.
+ Paths.clearTempFolderInfo(context.getTaskAttemptID());
+ }
+
+ LOG.debug("Committing wrapped task");
+ wrappedCommitter.commitTask(context);
+
+ LOG.debug("Cleaning up attempt dir {}", attemptPath);
+ attemptFS.delete(attemptPath, true);
+ return commits.size();
+ }
+
+ /**
+ * Abort the task.
+ * The API specifies that the task has not yet been committed, so there are
+ * no uploads that need to be cancelled.
+ * Accordingly just delete files on the local FS, and call abortTask in
+ * the wrapped committer.
+ * <b>Important: this may be called in the AM after a container failure.</b>
+ * When that occurs and the failed container was on a different host in the
+ * cluster, the local files will not be deleted.
+ * @param context task context
+ * @throws IOException any failure
+ */
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ // the API specifies that the task has not yet been committed, so there are
+ // no uploads that need to be cancelled. just delete files on the local FS.
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Abort task %s", context.getTaskAttemptID())) {
+ deleteTaskAttemptPathQuietly(context);
+ deleteTaskWorkingPathQuietly(context);
+ wrappedCommitter.abortTask(context);
+ } catch (IOException e) {
+ LOG.error("{}: exception when aborting task {}",
+ getRole(), context.getTaskAttemptID(), e);
+ throw e;
+ }
+ }
+
+ /**
+ * Get the work path for a task.
+ * @param context job/task complex
+ * @param uuid UUID
+ * @return a path
+ * @throws IOException failure to build the path
+ */
+ private static Path taskAttemptWorkingPath(TaskAttemptContext context,
+ String uuid) throws IOException {
+ return getTaskAttemptPath(context,
+ Paths.getLocalTaskAttemptTempDir(
+ context.getConfiguration(),
+ uuid,
+ context.getTaskAttemptID()));
+ }
+
+ /**
+ * Delete the working path of a task; no-op if there is none, that
+ * is: this is a job.
+ * @param context job/task context
+ */
+ protected void deleteTaskWorkingPathQuietly(JobContext context) {
+ ignoreIOExceptions(LOG, "Delete working path", "",
+ () -> {
+ Path path = buildWorkPath(context, getUUID());
+ if (path != null) {
+ deleteQuietly(path.getFileSystem(getConf()), path, true);
+ }
+ });
+ }
+
+ /**
+ * Get the key of the destination "directory" of the job/task.
+ * @param context job context
+ * @return key to write to
+ */
+ private String getS3KeyPrefix(JobContext context) {
+ return s3KeyPrefix;
+ }
+
+ /**
+ * A UUID for this upload, as calculated with.
+ * {@link #getUploadUUID(Configuration, String)}
+ * @return the UUID for files
+ */
+ protected String getUUID() {
+ return uuid;
+ }
+
+ /**
+ * Returns the {@link ConflictResolution} mode for this commit.
+ *
+ * @param context the JobContext for this commit
+ * @param fsConf filesystem config
+ * @return the ConflictResolution mode
+ */
+ public final ConflictResolution getConflictResolutionMode(
+ JobContext context,
+ Configuration fsConf) {
+ if (conflictResolution == null) {
+ this.conflictResolution = ConflictResolution.valueOf(
+ getConfictModeOption(context, fsConf));
+ }
+ return conflictResolution;
+ }
+
+ /**
+ * Get the conflict mode option string.
+ * @param context context with the config
+ * @param fsConf filesystem config
+ * @return the trimmed configuration option, upper case.
+ */
+ public static String getConfictModeOption(JobContext context,
+ Configuration fsConf) {
+ return getConfigurationOption(context,
+ fsConf,
+ FS_S3A_COMMITTER_STAGING_CONFLICT_MODE,
+ DEFAULT_CONFLICT_MODE).toUpperCase(Locale.ENGLISH);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterConstants.java
new file mode 100644
index 0000000..c5fb967
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterConstants.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Internal staging committer constants.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class StagingCommitterConstants {
+
+ private StagingCommitterConstants() {
+ }
+
+ /**
+ * The temporary path for staging data, if not explicitly set.
+ * By using an unqualified path, this will be qualified to be relative
+ * to the users' home directory, so protectec from access for others.
+ */
+ public static final String FILESYSTEM_TEMP_PATH = "tmp/staging";
+
+ /** Name of the root partition :{@value}. */
+ public static final String TABLE_ROOT = "table_root";
+
+ /**
+ * Filename used under {@code ~/${UUID}} for the staging files.
+ */
+ public static final String STAGING_UPLOADS = "staging-uploads";
+
+ // Spark configuration keys
+
+ /**
+ * The UUID for jobs: {@value}.
+ */
+ public static final String SPARK_WRITE_UUID =
+ "spark.sql.sources.writeJobUUID";
+
+ /**
+ * The App ID for jobs.
+ */
+
+ public static final String SPARK_APP_ID = "spark.app.id";
+
+ public static final String JAVA_IO_TMPDIR = "java.io.tmpdir";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterFactory.java
new file mode 100644
index 0000000..292b16b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/StagingCommitterFactory.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitterFactory;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+
+/**
+ * Factory for the staging committer.
+ * This is for internal test use, rather than the public directory and
+ * partitioned committers.
+ */
+public class StagingCommitterFactory
+ extends AbstractS3ACommitterFactory {
+
+ /**
+ * Name of this class: {@value}.
+ */
+ public static final String CLASSNAME
+ = "org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterFactory";
+
+ public PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem,
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new StagingCommitter(outputPath, context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/package-info.java
new file mode 100644
index 0000000..174fbb0
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The staging committers.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
index 458eb83..13384cf 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/DynamoDBMetadataStore.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import com.amazonaws.AmazonClientException;
import com.amazonaws.services.dynamodbv2.AmazonDynamoDB;
@@ -67,8 +68,12 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.Retries;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
+import org.apache.hadoop.fs.s3a.S3ARetryPolicy;
+import org.apache.hadoop.fs.s3a.S3AUtils;
import org.apache.hadoop.fs.s3a.Tristate;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
@@ -211,6 +216,28 @@ public class DynamoDBMetadataStore implements MetadataStore {
private RetryPolicy dataAccessRetryPolicy;
private S3AInstrumentation.S3GuardInstrumentation instrumentation;
+ /** Owner FS: only valid if configured with an owner FS. */
+ private S3AFileSystem owner;
+
+ /** Invoker for IO. Until configured properly, use try-once. */
+ private Invoker invoker = new Invoker(RetryPolicies.TRY_ONCE_THEN_FAIL,
+ Invoker.NO_OP
+ );
+
+ /** Data access can have its own policies. */
+ private Invoker dataAccess;
+
+ /**
+ * Total limit on the number of throttle events after which
+ * we stop warning in the log. Keeps the noise down.
+ */
+ private static final int THROTTLE_EVENT_LOG_LIMIT = 100;
+
+ /**
+ * Count of the total number of throttle events; used to crank back logging.
+ */
+ private AtomicInteger throttleEventCount = new AtomicInteger(0);
+
/**
* A utility function to create DynamoDB instance.
* @param conf the file system configuration
@@ -232,28 +259,34 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceRaw
public void initialize(FileSystem fs) throws IOException {
Preconditions.checkArgument(fs instanceof S3AFileSystem,
"DynamoDBMetadataStore only supports S3A filesystem.");
- final S3AFileSystem s3afs = (S3AFileSystem) fs;
- instrumentation = s3afs.getInstrumentation().getS3GuardInstrumentation();
- final String bucket = s3afs.getBucket();
- String confRegion = s3afs.getConf().getTrimmed(S3GUARD_DDB_REGION_KEY);
+ owner = (S3AFileSystem) fs;
+ instrumentation = owner.getInstrumentation().getS3GuardInstrumentation();
+ final String bucket = owner.getBucket();
+ conf = owner.getConf();
+ String confRegion = conf.getTrimmed(S3GUARD_DDB_REGION_KEY);
if (!StringUtils.isEmpty(confRegion)) {
region = confRegion;
LOG.debug("Overriding S3 region with configured DynamoDB region: {}",
region);
} else {
- region = s3afs.getBucketLocation();
+ region = owner.getBucketLocation();
LOG.debug("Inferring DynamoDB region from S3 bucket: {}", region);
}
- username = s3afs.getUsername();
- conf = s3afs.getConf();
+ username = owner.getUsername();
dynamoDB = createDynamoDB(conf, region);
// use the bucket as the DynamoDB table name if not specified in config
tableName = conf.getTrimmed(S3GUARD_DDB_TABLE_NAME_KEY, bucket);
- setMaxRetries(conf);
+ initDataAccessRetries(conf);
+
+ // set up a full retry policy
+ invoker = new Invoker(new S3ARetryPolicy(conf),
+ this::retryEvent
+ );
initTable();
@@ -283,6 +316,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @throws IllegalArgumentException if the configuration is incomplete
*/
@Override
+ @Retries.OnceRaw
public void initialize(Configuration config) throws IOException {
conf = config;
// use the bucket as the DynamoDB table name if not specified in config
@@ -295,7 +329,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
dynamoDB = createDynamoDB(conf, region);
username = UserGroupInformation.getCurrentUser().getShortUserName();
- setMaxRetries(conf);
+ initDataAccessRetries(conf);
initTable();
}
@@ -304,22 +338,25 @@ public class DynamoDBMetadataStore implements MetadataStore {
* Set retry policy. This is driven by the value of
* {@link Constants#S3GUARD_DDB_MAX_RETRIES} with an exponential backoff
* between each attempt of {@link #MIN_RETRY_SLEEP_MSEC} milliseconds.
- * @param config
+ * @param config configuration for data access
*/
- private void setMaxRetries(Configuration config) {
+ private void initDataAccessRetries(Configuration config) {
int maxRetries = config.getInt(S3GUARD_DDB_MAX_RETRIES,
S3GUARD_DDB_MAX_RETRIES_DEFAULT);
dataAccessRetryPolicy = RetryPolicies
.exponentialBackoffRetry(maxRetries, MIN_RETRY_SLEEP_MSEC,
TimeUnit.MILLISECONDS);
+ dataAccess = new Invoker(dataAccessRetryPolicy, this::retryEvent);
}
@Override
+ @Retries.RetryTranslated
public void delete(Path path) throws IOException {
innerDelete(path, true);
}
@Override
+ @Retries.RetryTranslated
public void forgetMetadata(Path path) throws IOException {
innerDelete(path, false);
}
@@ -332,9 +369,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @param tombstone flag to create a tombstone marker
* @throws IOException I/O error.
*/
- private void innerDelete(Path path, boolean tombstone)
+ @Retries.RetryTranslated
+ private void innerDelete(final Path path, boolean tombstone)
throws IOException {
- path = checkPath(path);
+ checkPath(path);
LOG.debug("Deleting from table {} in region {}: {}",
tableName, region, path);
@@ -343,23 +381,25 @@ public class DynamoDBMetadataStore implements MetadataStore {
LOG.debug("Skip deleting root directory as it does not exist in table");
return;
}
-
- try {
- if (tombstone) {
- Item item = PathMetadataDynamoDBTranslation.pathMetadataToItem(
- PathMetadata.tombstone(path));
- table.putItem(item);
- } else {
- table.deleteItem(pathToKey(path));
- }
- } catch (AmazonClientException e) {
- throw translateException("delete", path, e);
+ // the policy on whether repeating delete operations is based
+ // on that of S3A itself
+ boolean idempotent = S3AFileSystem.DELETE_CONSIDERED_IDEMPOTENT;
+ if (tombstone) {
+ Item item = PathMetadataDynamoDBTranslation.pathMetadataToItem(
+ PathMetadata.tombstone(path));
+ invoker.retry("Put tombstone", path.toString(), idempotent,
+ () -> table.putItem(item));
+ } else {
+ PrimaryKey key = pathToKey(path);
+ invoker.retry("Delete key", path.toString(), idempotent,
+ () -> table.deleteItem(key));
}
}
@Override
+ @Retries.RetryTranslated
public void deleteSubtree(Path path) throws IOException {
- path = checkPath(path);
+ checkPath(path);
LOG.debug("Deleting subtree from table {} in region {}: {}",
tableName, region, path);
@@ -375,6 +415,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
}
+ @Retries.OnceRaw
private Item getConsistentItem(PrimaryKey key) {
final GetItemSpec spec = new GetItemSpec()
.withPrimaryKey(key)
@@ -383,52 +424,65 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceTranslated
public PathMetadata get(Path path) throws IOException {
return get(path, false);
}
@Override
+ @Retries.OnceTranslated
public PathMetadata get(Path path, boolean wantEmptyDirectoryFlag)
throws IOException {
- path = checkPath(path);
+ checkPath(path);
LOG.debug("Get from table {} in region {}: {}", tableName, region, path);
+ return Invoker.once("get", path.toString(),
+ () -> innerGet(path, wantEmptyDirectoryFlag));
+ }
- try {
- final PathMetadata meta;
- if (path.isRoot()) {
- // Root does not persist in the table
- meta = new PathMetadata(makeDirStatus(username, path));
- } else {
- final Item item = getConsistentItem(pathToKey(path));
- meta = itemToPathMetadata(item, username);
- LOG.debug("Get from table {} in region {} returning for {}: {}",
- tableName, region, path, meta);
- }
+ /**
+ * Inner get operation, as invoked in the retry logic.
+ * @param path the path to get
+ * @param wantEmptyDirectoryFlag Set to true to give a hint to the
+ * MetadataStore that it should try to compute the empty directory flag.
+ * @return metadata for {@code path}, {@code null} if not found
+ * @throws IOException IO problem
+ * @throws AmazonClientException dynamo DB level problem
+ */
+ @Retries.OnceRaw
+ private PathMetadata innerGet(Path path, boolean wantEmptyDirectoryFlag)
+ throws IOException {
+ final PathMetadata meta;
+ if (path.isRoot()) {
+ // Root does not persist in the table
+ meta = new PathMetadata(makeDirStatus(username, path));
+ } else {
+ final Item item = getConsistentItem(pathToKey(path));
+ meta = itemToPathMetadata(item, username);
+ LOG.debug("Get from table {} in region {} returning for {}: {}",
+ tableName, region, path, meta);
+ }
- if (wantEmptyDirectoryFlag && meta != null) {
- final FileStatus status = meta.getFileStatus();
- // for directory, we query its direct children to determine isEmpty bit
- if (status.isDirectory()) {
- final QuerySpec spec = new QuerySpec()
- .withHashKey(pathToParentKeyAttribute(path))
- .withConsistentRead(true)
- .withFilterExpression(IS_DELETED + " = :false")
- .withValueMap(deleteTrackingValueMap);
- final ItemCollection<QueryOutcome> items = table.query(spec);
- boolean hasChildren = items.iterator().hasNext();
- // When this class has support for authoritative
- // (fully-cached) directory listings, we may also be able to answer
- // TRUE here. Until then, we don't know if we have full listing or
- // not, thus the UNKNOWN here:
- meta.setIsEmptyDirectory(
- hasChildren ? Tristate.FALSE : Tristate.UNKNOWN);
- }
+ if (wantEmptyDirectoryFlag && meta != null) {
+ final FileStatus status = meta.getFileStatus();
+ // for directory, we query its direct children to determine isEmpty bit
+ if (status.isDirectory()) {
+ final QuerySpec spec = new QuerySpec()
+ .withHashKey(pathToParentKeyAttribute(path))
+ .withConsistentRead(true)
+ .withFilterExpression(IS_DELETED + " = :false")
+ .withValueMap(deleteTrackingValueMap);
+ final ItemCollection<QueryOutcome> items = table.query(spec);
+ boolean hasChildren = items.iterator().hasNext();
+ // When this class has support for authoritative
+ // (fully-cached) directory listings, we may also be able to answer
+ // TRUE here. Until then, we don't know if we have full listing or
+ // not, thus the UNKNOWN here:
+ meta.setIsEmptyDirectory(
+ hasChildren ? Tristate.FALSE : Tristate.UNKNOWN);
}
-
- return meta;
- } catch (AmazonClientException e) {
- throw translateException("get", path, e);
}
+
+ return meta;
}
/**
@@ -445,35 +499,38 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
- public DirListingMetadata listChildren(Path path) throws IOException {
- path = checkPath(path);
+ @Retries.OnceTranslated
+ public DirListingMetadata listChildren(final Path path) throws IOException {
+ checkPath(path);
LOG.debug("Listing table {} in region {}: {}", tableName, region, path);
// find the children in the table
- try {
- final QuerySpec spec = new QuerySpec()
- .withHashKey(pathToParentKeyAttribute(path))
- .withConsistentRead(true); // strictly consistent read
- final ItemCollection<QueryOutcome> items = table.query(spec);
-
- final List<PathMetadata> metas = new ArrayList<>();
- for (Item item : items) {
- PathMetadata meta = itemToPathMetadata(item, username);
- metas.add(meta);
- }
- LOG.trace("Listing table {} in region {} for {} returning {}",
- tableName, region, path, metas);
+ return Invoker.once("listChildren", path.toString(),
+ () -> {
+ final QuerySpec spec = new QuerySpec()
+ .withHashKey(pathToParentKeyAttribute(path))
+ .withConsistentRead(true); // strictly consistent read
+ final ItemCollection<QueryOutcome> items = table.query(spec);
- return (metas.isEmpty() && get(path) == null)
- ? null
- : new DirListingMetadata(path, metas, false);
- } catch (AmazonClientException e) {
- // failure, including the path not being present
- throw translateException("listChildren", path, e);
- }
+ final List<PathMetadata> metas = new ArrayList<>();
+ for (Item item : items) {
+ PathMetadata meta = itemToPathMetadata(item, username);
+ metas.add(meta);
+ }
+ LOG.trace("Listing table {} in region {} for {} returning {}",
+ tableName, region, path, metas);
+
+ return (metas.isEmpty() && get(path) == null)
+ ? null
+ : new DirListingMetadata(path, metas, false);
+ });
}
- // build the list of all parent entries.
+ /**
+ * build the list of all parent entries.
+ * @param pathsToCreate paths to create
+ * @return the full ancestry paths
+ */
Collection<PathMetadata> completeAncestry(
Collection<PathMetadata> pathsToCreate) {
// Key on path to allow fast lookup
@@ -499,6 +556,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceTranslated
public void move(Collection<Path> pathsToDelete,
Collection<PathMetadata> pathsToCreate) throws IOException {
if (pathsToDelete == null && pathsToCreate == null) {
@@ -527,21 +585,20 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
}
- try {
- processBatchWriteRequest(null, pathMetadataToItem(newItems));
- } catch (AmazonClientException e) {
- throw translateException("move", (String) null, e);
- }
+ Invoker.once("move", tableName,
+ () -> processBatchWriteRequest(null, pathMetadataToItem(newItems)));
}
/**
* Helper method to issue a batch write request to DynamoDB.
*
- * Callers of this method should catch the {@link AmazonClientException} and
- * translate it for better error report and easier debugging.
+ * The retry logic here is limited to repeating the write operations
+ * until all items have been written; there is no other attempt
+ * at recovery/retry. Throttling is handled internally.
* @param keysToDelete primary keys to be deleted; can be null
* @param itemsToPut new items to be put; can be null
*/
+ @Retries.OnceRaw("Outstanding batch items are updated with backoff")
private void processBatchWriteRequest(PrimaryKey[] keysToDelete,
Item[] itemsToPut) throws IOException {
final int totalToDelete = (keysToDelete == null ? 0 : keysToDelete.length);
@@ -575,7 +632,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
// Check for unprocessed keys in case of exceeding provisioned throughput
Map<String, List<WriteRequest>> unprocessed = res.getUnprocessedItems();
int retryCount = 0;
- while (unprocessed.size() > 0) {
+ while (!unprocessed.isEmpty()) {
retryBackoff(retryCount++);
res = dynamoDB.batchWriteItemUnprocessed(unprocessed);
unprocessed = res.getUnprocessedItems();
@@ -603,12 +660,17 @@ public class DynamoDBMetadataStore implements MetadataStore {
LOG.debug("Sleeping {} msec before next retry", action.delayMillis);
Thread.sleep(action.delayMillis);
}
+ } catch (InterruptedException e) {
+ throw (IOException)new InterruptedIOException(e.toString()).initCause(e);
+ } catch (IOException e) {
+ throw e;
} catch (Exception e) {
throw new IOException("Unexpected exception", e);
}
}
@Override
+ @Retries.OnceRaw
public void put(PathMetadata meta) throws IOException {
// For a deeply nested path, this method will automatically create the full
// ancestry and save respective item in DynamoDB table.
@@ -624,6 +686,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceRaw
public void put(Collection<PathMetadata> metas) throws IOException {
LOG.debug("Saving batch to table {} in region {}", tableName, region);
@@ -633,6 +696,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
/**
* Helper method to get full path of ancestors that are nonexistent in table.
*/
+ @Retries.OnceRaw
private Collection<PathMetadata> fullPathsToPut(PathMetadata meta)
throws IOException {
checkPathMetadata(meta);
@@ -675,25 +739,34 @@ public class DynamoDBMetadataStore implements MetadataStore {
null, owner, owner, f);
}
+ /**
+ * {@inheritDoc}.
+ * There is retry around building the list of paths to update, but
+ * the call to {@link #processBatchWriteRequest(PrimaryKey[], Item[])}
+ * is only tried once.
+ * @param meta Directory listing metadata.
+ * @throws IOException
+ */
@Override
+ @Retries.OnceTranslated("retry(listFullPaths); once(batchWrite)")
public void put(DirListingMetadata meta) throws IOException {
LOG.debug("Saving to table {} in region {}: {}", tableName, region, meta);
// directory path
- PathMetadata p = new PathMetadata(makeDirStatus(meta.getPath(), username),
+ Path path = meta.getPath();
+ PathMetadata p = new PathMetadata(makeDirStatus(path, username),
meta.isEmpty(), false);
// First add any missing ancestors...
- final Collection<PathMetadata> metasToPut = fullPathsToPut(p);
+ final Collection<PathMetadata> metasToPut = invoker.retry(
+ "paths to put", path.toString(), true,
+ () -> fullPathsToPut(p));
// next add all children of the directory
metasToPut.addAll(meta.getListing());
- try {
- processBatchWriteRequest(null, pathMetadataToItem(metasToPut));
- } catch (AmazonClientException e) {
- throw translateException("put", (String) null, e);
- }
+ Invoker.once("put", path.toString(),
+ () -> processBatchWriteRequest(null, pathMetadataToItem(metasToPut)));
}
@Override
@@ -709,6 +782,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceTranslated
public void destroy() throws IOException {
if (table == null) {
LOG.info("In destroy(): no table to delete");
@@ -731,10 +805,11 @@ public class DynamoDBMetadataStore implements MetadataStore {
throw new InterruptedIOException("Table " + tableName
+ " in region " + region + " has not been deleted");
} catch (AmazonClientException e) {
- throw translateException("destroy", (String) null, e);
+ throw translateException("destroy", tableName, e);
}
}
+ @Retries.OnceRaw
private ItemCollection<ScanOutcome> expiredFiles(long modTime) {
String filterExpression = "mod_time < :mod_time";
String projectionExpression = "parent,child";
@@ -743,6 +818,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceRaw("once(batchWrite)")
public void prune(long modTime) throws IOException {
int itemCount = 0;
try {
@@ -797,6 +873,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
* or table is being deleted, or any other I/O exception occurred.
*/
@VisibleForTesting
+ @Retries.OnceRaw
void initTable() throws IOException {
table = dynamoDB.getTable(tableName);
try {
@@ -848,7 +925,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
} catch (AmazonClientException e) {
- throw translateException("initTable", (String) null, e);
+ throw translateException("initTable", tableName, e);
}
}
@@ -856,8 +933,11 @@ public class DynamoDBMetadataStore implements MetadataStore {
* Get the version mark item in the existing DynamoDB table.
*
* As the version marker item may be created by another concurrent thread or
- * process, we retry a limited times before we fail to get it.
+ * process, we sleep and retry a limited times before we fail to get it.
+ * This does not include handling any failure other than "item not found",
+ * so this method is tagged as "OnceRaw"
*/
+ @Retries.OnceRaw
private Item getVersionMarkerItem() throws IOException {
final PrimaryKey versionMarkerKey =
createVersionMarkerPrimaryKey(VERSION_MARKER);
@@ -913,16 +993,20 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @param t table to block on.
* @throws IOException IO problems
* @throws InterruptedIOException if the wait was interrupted
+ * @throws IllegalArgumentException if an exception was raised in the waiter
*/
- private void waitForTableActive(Table t) throws IOException {
+ @Retries.OnceRaw
+ private void waitForTableActive(Table t) throws InterruptedIOException {
try {
t.waitForActive();
} catch (InterruptedException e) {
LOG.warn("Interrupted while waiting for table {} in region {} active",
tableName, region, e);
Thread.currentThread().interrupt();
- throw (IOException) new InterruptedIOException("DynamoDB table '"
- + tableName + "' is not active yet in region " + region).initCause(e);
+ throw (InterruptedIOException)
+ new InterruptedIOException("DynamoDB table '"
+ + tableName + "' is not active yet in region " + region)
+ .initCause(e);
}
}
@@ -933,6 +1017,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @throws IOException on any failure.
* @throws InterruptedIOException if the wait was interrupted
*/
+ @Retries.OnceRaw
private void createTable(ProvisionedThroughput capacity) throws IOException {
try {
LOG.info("Creating non-existent DynamoDB table {} in region {}",
@@ -960,6 +1045,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
* @param item item to put
* @return the outcome.
*/
+ @Retries.OnceRaw
PutItemOutcome putItem(Item item) {
LOG.debug("Putting item {}", item);
return table.putItem(item);
@@ -967,22 +1053,27 @@ public class DynamoDBMetadataStore implements MetadataStore {
/**
* Provision the table with given read and write capacity units.
+ * Call will fail if the table is busy, or the new values match the current
+ * ones.
+ * @param readCapacity read units
+ * @param writeCapacity write units
+ * @throws IOException on a failure
*/
+ @Retries.RetryTranslated
void provisionTable(Long readCapacity, Long writeCapacity)
throws IOException {
final ProvisionedThroughput toProvision = new ProvisionedThroughput()
.withReadCapacityUnits(readCapacity)
.withWriteCapacityUnits(writeCapacity);
- try {
- final ProvisionedThroughputDescription p =
- table.updateTable(toProvision).getProvisionedThroughput();
- LOG.info("Provision table {} in region {}: readCapacityUnits={}, "
- + "writeCapacityUnits={}",
- tableName, region, p.getReadCapacityUnits(),
- p.getWriteCapacityUnits());
- } catch (AmazonClientException e) {
- throw translateException("provisionTable", (String) null, e);
- }
+ invoker.retry("ProvisionTable", tableName, true,
+ () -> {
+ final ProvisionedThroughputDescription p =
+ table.updateTable(toProvision).getProvisionedThroughput();
+ LOG.info("Provision table {} in region {}: readCapacityUnits={}, "
+ + "writeCapacityUnits={}",
+ tableName, region, p.getReadCapacityUnits(),
+ p.getWriteCapacityUnits());
+ });
}
Table getTable() {
@@ -999,8 +1090,10 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
/**
- * Validates a path object; it must be absolute, and contain a host
- * (bucket) component.
+ * Validates a path object; it must be absolute, have an s3a:/// scheme
+ * and contain a host (bucket) component.
+ * @param path path to check
+ * @return the path passed in
*/
private Path checkPath(Path path) {
Preconditions.checkNotNull(path);
@@ -1025,6 +1118,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceRaw
public Map<String, String> getDiagnostics() throws IOException {
Map<String, String> map = new TreeMap<>();
if (table != null) {
@@ -1052,6 +1146,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
return map;
}
+ @Retries.OnceRaw
private TableDescription getTableDescription(boolean forceUpdate) {
TableDescription desc = table.getDescription();
if (desc == null || forceUpdate) {
@@ -1061,6 +1156,7 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
@Override
+ @Retries.OnceRaw
public void updateParameters(Map<String, String> parameters)
throws IOException {
Preconditions.checkNotNull(table, "Not initialized");
@@ -1103,4 +1199,46 @@ public class DynamoDBMetadataStore implements MetadataStore {
}
}
+ /**
+ * Callback from {@link Invoker} when an operation is retried.
+ * @param text text of the operation
+ * @param ex exception
+ * @param attempts number of attempts
+ * @param idempotent is the method idempotent
+ */
+ void retryEvent(
+ String text,
+ IOException ex,
+ int attempts,
+ boolean idempotent) {
+ if (S3AUtils.isThrottleException(ex)) {
+ // throttled
+ if (instrumentation != null) {
+ instrumentation.throttled();
+ }
+ int eventCount = throttleEventCount.addAndGet(1);
+ if (attempts == 1 && eventCount < THROTTLE_EVENT_LOG_LIMIT) {
+ LOG.warn("DynamoDB IO limits reached in {};"
+ + " consider increasing capacity: {}", text, ex.toString());
+ LOG.debug("Throttled", ex);
+ } else {
+ // user has been warned already, log at debug only.
+ LOG.debug("DynamoDB IO limits reached in {};"
+ + " consider increasing capacity: {}", text, ex.toString());
+ }
+ } else if (attempts == 1) {
+ // not throttled. Log on the first attempt only
+ LOG.info("Retrying {}: {}", text, ex.toString());
+ LOG.debug("Retrying {}", text, ex);
+ }
+
+ if (instrumentation != null) {
+ // note a retry
+ instrumentation.retrying();
+ }
+ if (owner != null) {
+ owner.metastoreOperationRetried(ex, attempts, idempotent);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
index c7c810a..a56b055 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3Guard.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Retries;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AInstrumentation;
import org.apache.hadoop.fs.s3a.Tristate;
@@ -83,6 +84,7 @@ public final class S3Guard {
* @return Reference to new MetadataStore.
* @throws IOException if the metadata store cannot be instantiated
*/
+ @Retries.OnceTranslated
public static MetadataStore getMetadataStore(FileSystem fs)
throws IOException {
Preconditions.checkNotNull(fs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
index 4f0e8f7..ace043b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/s3guard/S3GuardTool.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.S3AFileStatus;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.GenericOptionsParser;
@@ -960,6 +961,7 @@ public abstract class S3GuardTool extends Configured implements Tool {
public static final String AUTH_FLAG = "auth";
public static final String NONAUTH_FLAG = "nonauth";
public static final String ENCRYPTION_FLAG = "encryption";
+ public static final String MAGIC_FLAG = "magic";
public static final String PURPOSE = "provide/check S3Guard information"
+ " about a specific bucket";
@@ -967,11 +969,15 @@ public abstract class S3GuardTool extends Configured implements Tool {
+ "\t" + PURPOSE + "\n\n"
+ "Common options:\n"
+ " -" + GUARDED_FLAG + " - Require S3Guard\n"
+ + " -" + UNGUARDED_FLAG + " - Require S3Guard to be disabled\n"
+ + " -" + AUTH_FLAG + " - Require the S3Guard mode to be \"authoritative\"\n"
+ + " -" + NONAUTH_FLAG + " - Require the S3Guard mode to be \"non-authoritative\"\n"
+ + " -" + MAGIC_FLAG + " - Require the S3 filesystem to be support the \"magic\" committer\n"
+ " -" + ENCRYPTION_FLAG
+ " -require {none, sse-s3, sse-kms} - Require encryption policy";
BucketInfo(Configuration conf) {
- super(conf, GUARDED_FLAG, UNGUARDED_FLAG, AUTH_FLAG, NONAUTH_FLAG);
+ super(conf, GUARDED_FLAG, UNGUARDED_FLAG, AUTH_FLAG, NONAUTH_FLAG, MAGIC_FLAG);
CommandFormat format = getCommandFormat();
format.addOptionWithValue(ENCRYPTION_FLAG);
}
@@ -1014,6 +1020,11 @@ public abstract class S3GuardTool extends Configured implements Tool {
} else {
println(out, "Filesystem %s is not using S3Guard", fsUri);
}
+ boolean magic = fs.hasCapability(
+ CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER);
+ println(out, "The \"magic\" committer %s supported",
+ magic ? "is" : "is not");
+
println(out, "%nS3A Client");
String endpoint = conf.getTrimmed(ENDPOINT, "");
@@ -1043,6 +1054,9 @@ public abstract class S3GuardTool extends Configured implements Tool {
throw badState("S3Guard is not enabled for %s", fsUri);
}
}
+ if (commands.getOpt(MAGIC_FLAG) && !magic) {
+ throw badState("The magic committer is not enabled for %s", fsUri);
+ }
String desiredEncryption = getCommandFormat()
.getOptValue(ENCRYPTION_FLAG);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java
new file mode 100644
index 0000000..b6b6b97
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Tasks.java
@@ -0,0 +1,410 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for parallel execution, takes closures for the various
+ * actions.
+ * There is no retry logic: it is expected to be handled by the closures.
+ */
+public final class Tasks {
+ private static final Logger LOG = LoggerFactory.getLogger(Tasks.class);
+
+ private Tasks() {
+ }
+
+ /**
+ * Callback invoked to process an item.
+ * @param <I> item type being processed
+ * @param <E> exception class which may be raised
+ */
+ @FunctionalInterface
+ public interface Task<I, E extends Exception> {
+ void run(I item) throws E;
+ }
+
+ /**
+ * Callback invoked on a failure.
+ * @param <I> item type being processed
+ * @param <E> exception class which may be raised
+ */
+ @FunctionalInterface
+ public interface FailureTask<I, E extends Exception> {
+
+ /**
+ * process a failure.
+ * @param item item the task is processing
+ * @param exception the exception which was raised.
+ * @throws E Exception of type E
+ */
+ void run(I item, Exception exception) throws E;
+ }
+
+ /**
+ * Builder for task execution.
+ * @param <I> item type
+ */
+ public static class Builder<I> {
+ private final Iterable<I> items;
+ private ExecutorService service = null;
+ private FailureTask<I, ?> onFailure = null;
+ private boolean stopOnFailure = false;
+ private boolean suppressExceptions = false;
+ private Task<I, ?> revertTask = null;
+ private boolean stopRevertsOnFailure = false;
+ private Task<I, ?> abortTask = null;
+ private boolean stopAbortsOnFailure = false;
+
+ /**
+ * Create the builder.
+ * @param items items to process
+ */
+ Builder(Iterable<I> items) {
+ this.items = items;
+ }
+
+ /**
+ * Declare executor service: if null, the tasks are executed in a single
+ * thread.
+ * @param executorService service to schedule tasks with.
+ * @return this builder.
+ */
+ public Builder<I> executeWith(ExecutorService executorService) {
+ this.service = executorService;
+ return this;
+ }
+
+ public Builder<I> onFailure(FailureTask<I, ?> task) {
+ this.onFailure = task;
+ return this;
+ }
+
+ public Builder<I> stopOnFailure() {
+ this.stopOnFailure = true;
+ return this;
+ }
+
+ public Builder<I> suppressExceptions() {
+ return suppressExceptions(true);
+ }
+
+ public Builder<I> suppressExceptions(boolean suppress) {
+ this.suppressExceptions = suppress;
+ return this;
+ }
+
+ public Builder<I> revertWith(Task<I, ?> task) {
+ this.revertTask = task;
+ return this;
+ }
+
+ public Builder<I> stopRevertsOnFailure() {
+ this.stopRevertsOnFailure = true;
+ return this;
+ }
+
+ public Builder<I> abortWith(Task<I, ?> task) {
+ this.abortTask = task;
+ return this;
+ }
+
+ public Builder<I> stopAbortsOnFailure() {
+ this.stopAbortsOnFailure = true;
+ return this;
+ }
+
+ public <E extends Exception> boolean run(Task<I, E> task) throws E {
+ if (service != null) {
+ return runParallel(task);
+ } else {
+ return runSingleThreaded(task);
+ }
+ }
+
+ private <E extends Exception> boolean runSingleThreaded(Task<I, E> task)
+ throws E {
+ List<I> succeeded = new ArrayList<>();
+ List<Exception> exceptions = new ArrayList<>();
+
+ Iterator<I> iterator = items.iterator();
+ boolean threw = true;
+ try {
+ while (iterator.hasNext()) {
+ I item = iterator.next();
+ try {
+ task.run(item);
+ succeeded.add(item);
+
+ } catch (Exception e) {
+ exceptions.add(e);
+
+ if (onFailure != null) {
+ try {
+ onFailure.run(item, e);
+ } catch (Exception failException) {
+ LOG.error("Failed to clean up on failure", e);
+ // keep going
+ }
+ }
+
+ if (stopOnFailure) {
+ break;
+ }
+ }
+ }
+
+ threw = false;
+
+ } finally {
+ // threw handles exceptions that were *not* caught by the catch block,
+ // and exceptions that were caught and possibly handled by onFailure
+ // are kept in exceptions.
+ if (threw || !exceptions.isEmpty()) {
+ if (revertTask != null) {
+ boolean failed = false;
+ for (I item : succeeded) {
+ try {
+ revertTask.run(item);
+ } catch (Exception e) {
+ LOG.error("Failed to revert task", e);
+ failed = true;
+ // keep going
+ }
+ if (stopRevertsOnFailure && failed) {
+ break;
+ }
+ }
+ }
+
+ if (abortTask != null) {
+ boolean failed = false;
+ while (iterator.hasNext()) {
+ try {
+ abortTask.run(iterator.next());
+ } catch (Exception e) {
+ failed = true;
+ LOG.error("Failed to abort task", e);
+ // keep going
+ }
+ if (stopAbortsOnFailure && failed) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ if (!suppressExceptions && !exceptions.isEmpty()) {
+ Tasks.<E>throwOne(exceptions);
+ }
+
+ return !threw && exceptions.isEmpty();
+ }
+
+ private <E extends Exception> boolean runParallel(final Task<I, E> task)
+ throws E {
+ final Queue<I> succeeded = new ConcurrentLinkedQueue<>();
+ final Queue<Exception> exceptions = new ConcurrentLinkedQueue<>();
+ final AtomicBoolean taskFailed = new AtomicBoolean(false);
+ final AtomicBoolean abortFailed = new AtomicBoolean(false);
+ final AtomicBoolean revertFailed = new AtomicBoolean(false);
+
+ List<Future<?>> futures = new ArrayList<>();
+
+ for (final I item : items) {
+ // submit a task for each item that will either run or abort the task
+ futures.add(service.submit(new Runnable() {
+ @Override
+ public void run() {
+ if (!(stopOnFailure && taskFailed.get())) {
+ // run the task
+ boolean threw = true;
+ try {
+ LOG.debug("Executing task");
+ task.run(item);
+ succeeded.add(item);
+ LOG.debug("Task succeeded");
+
+ threw = false;
+
+ } catch (Exception e) {
+ taskFailed.set(true);
+ exceptions.add(e);
+ LOG.info("Task failed", e);
+
+ if (onFailure != null) {
+ try {
+ onFailure.run(item, e);
+ } catch (Exception failException) {
+ LOG.error("Failed to clean up on failure", e);
+ // swallow the exception
+ }
+ }
+ } finally {
+ if (threw) {
+ taskFailed.set(true);
+ }
+ }
+
+ } else if (abortTask != null) {
+ // abort the task instead of running it
+ if (stopAbortsOnFailure && abortFailed.get()) {
+ return;
+ }
+
+ boolean failed = true;
+ try {
+ LOG.info("Aborting task");
+ abortTask.run(item);
+ failed = false;
+ } catch (Exception e) {
+ LOG.error("Failed to abort task", e);
+ // swallow the exception
+ } finally {
+ if (failed) {
+ abortFailed.set(true);
+ }
+ }
+ }
+ }
+ }));
+ }
+
+ // let the above tasks complete (or abort)
+ waitFor(futures);
+ int futureCount = futures.size();
+ futures.clear();
+
+ if (taskFailed.get() && revertTask != null) {
+ // at least one task failed, revert any that succeeded
+ LOG.info("Reverting all {} succeeded tasks from {} futures",
+ succeeded.size(), futureCount);
+ for (final I item : succeeded) {
+ futures.add(service.submit(() -> {
+ if (stopRevertsOnFailure && revertFailed.get()) {
+ return;
+ }
+
+ boolean failed = true;
+ try {
+ revertTask.run(item);
+ failed = false;
+ } catch (Exception e) {
+ LOG.error("Failed to revert task", e);
+ // swallow the exception
+ } finally {
+ if (failed) {
+ revertFailed.set(true);
+ }
+ }
+ }));
+ }
+
+ // let the revert tasks complete
+ waitFor(futures);
+ }
+
+ if (!suppressExceptions && !exceptions.isEmpty()) {
+ Tasks.<E>throwOne(exceptions);
+ }
+
+ return !taskFailed.get();
+ }
+ }
+
+ /**
+ * Wait for all the futures to complete; there's a small sleep between
+ * each iteration; enough to yield the CPU.
+ * @param futures futures.
+ */
+ private static void waitFor(Collection<Future<?>> futures) {
+ int size = futures.size();
+ LOG.debug("Waiting for {} tasks to complete", size);
+ int oldNumFinished = 0;
+ while (true) {
+ int numFinished = (int) futures.stream().filter(Future::isDone).count();
+
+ if (oldNumFinished != numFinished) {
+ LOG.debug("Finished count -> {}/{}", numFinished, size);
+ oldNumFinished = numFinished;
+ }
+
+ if (numFinished == size) {
+ // all of the futures are done, stop looping
+ break;
+ } else {
+ try {
+ Thread.sleep(10);
+ } catch (InterruptedException e) {
+ futures.forEach(future -> future.cancel(true));
+ Thread.currentThread().interrupt();
+ break;
+ }
+ }
+ }
+ }
+
+ public static <I> Builder<I> foreach(Iterable<I> items) {
+ return new Builder<>(items);
+ }
+
+ public static <I> Builder<I> foreach(I[] items) {
+ return new Builder<>(Arrays.asList(items));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <E extends Exception> void throwOne(
+ Collection<Exception> exceptions)
+ throws E {
+ Iterator<Exception> iter = exceptions.iterator();
+ Exception e = iter.next();
+ Class<? extends Exception> exceptionClass = e.getClass();
+
+ while (iter.hasNext()) {
+ Exception other = iter.next();
+ if (!exceptionClass.isInstance(other)) {
+ e.addSuppressed(other);
+ }
+ }
+
+ Tasks.<E>castAndThrow(e);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <E extends Exception> void castAndThrow(Exception e) throws E {
+ if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ }
+ throw (E) e;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/ValidationFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/ValidationFailure.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/ValidationFailure.java
new file mode 100644
index 0000000..ac29932
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/ValidationFailure.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+
+/**
+ * Exception raised on validation failures; kept as an IOException
+ * for consistency with other failures.
+ */
+public class ValidationFailure extends IOException {
+
+ /**
+ * Create an instance with string formatting applied to the message
+ * and arguments.
+ * @param message error message
+ * @param args optional list of arguments
+ */
+ public ValidationFailure(String message, Object... args) {
+ super(String.format(message, args));
+ }
+
+ /**
+ * Verify that a condition holds.
+ * @param expression expression which must be true
+ * @param message message to raise on a failure
+ * @param args arguments for the message formatting
+ * @throws ValidationFailure on a failure
+ */
+ public static void verify(boolean expression,
+ String message,
+ Object... args) throws ValidationFailure {
+ if (!expression) {
+ throw new ValidationFailure(message, args);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java
new file mode 100644
index 0000000..c0d7415
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PendingSet.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.files;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitUtils.validateCollectionClass;
+import static org.apache.hadoop.fs.s3a.commit.ValidationFailure.verify;
+
+/**
+ * Persistent format for multiple pending commits.
+ * Contains 0 or more {@link SinglePendingCommit} entries; validation logic
+ * checks those values on load.
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class PendingSet extends PersistentCommitData {
+ private static final Logger LOG = LoggerFactory.getLogger(PendingSet.class);
+
+ /**
+ * Supported version value: {@value}.
+ * If this is changed the value of {@link #serialVersionUID} will change,
+ * to avoid deserialization problems.
+ */
+ public static final int VERSION = 1;
+
+ /**
+ * Serialization ID: {@value}.
+ */
+ private static final long serialVersionUID = 0x11000 + VERSION;
+
+
+ /** Version marker. */
+ private int version = VERSION;
+
+ /**
+ * Commit list.
+ */
+ private List<SinglePendingCommit> commits;
+
+ /**
+ * Any custom extra data committer subclasses may choose to add.
+ */
+ private final Map<String, String> extraData = new HashMap<>(0);
+
+ public PendingSet() {
+ this(0);
+ }
+
+
+ public PendingSet(int size) {
+ commits = new ArrayList<>(size);
+ }
+
+ /**
+ * Get a JSON serializer for this class.
+ * @return a serializer.
+ */
+ public static JsonSerialization<PendingSet> serializer() {
+ return new JsonSerialization<>(PendingSet.class, false, true);
+ }
+
+ /**
+ * Load an instance from a file, then validate it.
+ * @param fs filesystem
+ * @param path path
+ * @return the loaded instance
+ * @throws IOException IO failure
+ * @throws ValidationFailure if the data is invalid
+ */
+ public static PendingSet load(FileSystem fs, Path path)
+ throws IOException {
+ LOG.debug("Reading pending commits in file {}", path);
+ PendingSet instance = serializer().load(fs, path);
+ instance.validate();
+ return instance;
+ }
+
+ /**
+ * Add a commit.
+ * @param commit the single commit
+ */
+ public void add(SinglePendingCommit commit) {
+ commits.add(commit);
+ }
+
+ /**
+ * Deserialize via java Serialization API: deserialize the instance
+ * and then call {@link #validate()} to verify that the deserialized
+ * data is valid.
+ * @param inStream input stream
+ * @throws IOException IO problem or validation failure
+ * @throws ClassNotFoundException reflection problems
+ */
+ private void readObject(ObjectInputStream inStream) throws IOException,
+ ClassNotFoundException {
+ inStream.defaultReadObject();
+ validate();
+ }
+
+ /**
+ * Validate the data: those fields which must be non empty, must be set.
+ * @throws ValidationFailure if the data is invalid
+ */
+ public void validate() throws ValidationFailure {
+ verify(version == VERSION, "Wrong version: %s", version);
+ validateCollectionClass(extraData.keySet(), String.class);
+ validateCollectionClass(extraData.values(), String.class);
+ Set<String> destinations = new HashSet<>(commits.size());
+ validateCollectionClass(commits, SinglePendingCommit.class);
+ for (SinglePendingCommit c : commits) {
+ c.validate();
+ verify(!destinations.contains(c.getDestinationKey()),
+ "Destination %s is written to by more than one pending commit",
+ c.getDestinationKey());
+ destinations.add(c.getDestinationKey());
+ }
+ }
+
+ @Override
+ public byte[] toBytes() throws IOException {
+ return serializer().toBytes(this);
+ }
+
+ /**
+ * Number of commits.
+ * @return the number of commits in this structure.
+ */
+ public int size() {
+ return commits != null ? commits.size() : 0;
+ }
+
+ @Override
+ public void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException {
+ serializer().save(fs, path, this, overwrite);
+ }
+
+ /** @return the version marker. */
+ public int getVersion() {
+ return version;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ /**
+ * @return commit list.
+ */
+ public List<SinglePendingCommit> getCommits() {
+ return commits;
+ }
+
+ public void setCommits(List<SinglePendingCommit> commits) {
+ this.commits = commits;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java
new file mode 100644
index 0000000..cc27d07
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/PersistentCommitData.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.files;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+
+/**
+ * Class for single/multiple commit data structures.
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class PersistentCommitData implements Serializable {
+
+ /**
+ * Supported version value: {@value}.
+ * If this is changed the value of {@code serialVersionUID} will change,
+ * to avoid deserialization problems.
+ */
+ public static final int VERSION = 1;
+
+ /**
+ * Validate the data: those fields which must be non empty, must be set.
+ * @throws ValidationFailure if the data is invalid
+ */
+ public abstract void validate() throws ValidationFailure;
+
+ /**
+ * Serialize to JSON and then to a byte array, after performing a
+ * preflight validation of the data to be saved.
+ * @return the data in a persistable form.
+ * @throws IOException serialization problem or validation failure.
+ */
+ public abstract byte[] toBytes() throws IOException;
+
+ /**
+ * Save to a hadoop filesystem.
+ * @param fs filesystem
+ * @param path path
+ * @param overwrite should any existing file be overwritten
+ * @throws IOException IO exception
+ */
+ public abstract void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
new file mode 100644
index 0000000..85cc38a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SinglePendingCommit.java
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.files;
+
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import com.amazonaws.services.s3.model.PartETag;
+import com.google.common.base.Preconditions;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.util.JsonSerialization;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitUtils.validateCollectionClass;
+import static org.apache.hadoop.fs.s3a.commit.ValidationFailure.verify;
+import static org.apache.hadoop.util.StringUtils.join;
+
+/**
+ * This is the serialization format for uploads yet to be committerd.
+ *
+ * It's marked as {@link Serializable} so that it can be passed in RPC
+ * calls; for this to work it relies on the fact that java.io ArrayList
+ * and LinkedList are serializable. If any other list type is used for etags,
+ * it must also be serialized. Jackson expects lists, and it is used
+ * to persist to disk.
+ *
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class SinglePendingCommit extends PersistentCommitData
+ implements Iterable<String> {
+
+ /**
+ * Serialization ID: {@value}.
+ */
+ private static final long serialVersionUID = 0x10000 + VERSION;
+
+ /** Version marker. */
+ private int version = VERSION;
+
+ /**
+ * This is the filename of the pending file itself.
+ * Used during processing; it's persistent value, if any, is ignored.
+ */
+ private String filename;
+
+ /** Path URI of the destination. */
+ private String uri = "";
+
+ /** ID of the upload. */
+ private String uploadId;
+
+ /** Destination bucket. */
+ private String bucket;
+
+ /** Destination key in the bucket. */
+ private String destinationKey;
+
+ /** When was the upload created? */
+ private long created;
+
+ /** When was the upload saved? */
+ private long saved;
+
+ /** timestamp as date; no expectation of parseability. */
+ private String date;
+
+ /** Job ID, if known. */
+ private String jobId = "";
+
+ /** Task ID, if known. */
+ private String taskId = "";
+
+ /** Arbitrary notes. */
+ private String text = "";
+
+ /** Ordered list of etags. */
+ private List<String> etags;
+
+ /**
+ * Any custom extra data committer subclasses may choose to add.
+ */
+ private Map<String, String> extraData = new HashMap<>(0);
+
+ /** Destination file size. */
+ private long length;
+
+ public SinglePendingCommit() {
+ }
+
+ /**
+ * Get a JSON serializer for this class.
+ * @return a serializer.
+ */
+ public static JsonSerialization<SinglePendingCommit> serializer() {
+ return new JsonSerialization<>(SinglePendingCommit.class, false, true);
+ }
+
+ /**
+ * Load an instance from a file, then validate it.
+ * @param fs filesystem
+ * @param path path
+ * @return the loaded instance
+ * @throws IOException IO failure
+ * @throws ValidationFailure if the data is invalid
+ */
+ public static SinglePendingCommit load(FileSystem fs, Path path)
+ throws IOException {
+ SinglePendingCommit instance = serializer().load(fs, path);
+ instance.filename = path.toString();
+ instance.validate();
+ return instance;
+ }
+
+ /**
+ * Deserialize via java Serialization API: deserialize the instance
+ * and then call {@link #validate()} to verify that the deserialized
+ * data is valid.
+ * @param inStream input stream
+ * @throws IOException IO problem
+ * @throws ClassNotFoundException reflection problems
+ * @throws ValidationFailure validation failure
+ */
+ private void readObject(ObjectInputStream inStream) throws IOException,
+ ClassNotFoundException {
+ inStream.defaultReadObject();
+ validate();
+ }
+
+ /**
+ * Set the various timestamp fields to the supplied value.
+ * @param millis time in milliseconds
+ */
+ public void touch(long millis) {
+ created = millis;
+ saved = millis;
+ date = new Date(millis).toString();
+ }
+
+ /**
+ * Set the commit data.
+ * @param parts ordered list of etags.
+ * @throws ValidationFailure if the data is invalid
+ */
+ public void bindCommitData(List<PartETag> parts) throws ValidationFailure {
+ etags = new ArrayList<>(parts.size());
+ int counter = 1;
+ for (PartETag part : parts) {
+ verify(part.getPartNumber() == counter,
+ "Expected part number %s but got %s", counter, part.getPartNumber());
+ etags.add(part.getETag());
+ counter++;
+ }
+ }
+
+ @Override
+ public void validate() throws ValidationFailure {
+ verify(version == VERSION, "Wrong version: %s", version);
+ verify(StringUtils.isNotEmpty(bucket), "Empty bucket");
+ verify(StringUtils.isNotEmpty(destinationKey),
+ "Empty destination");
+ verify(StringUtils.isNotEmpty(uploadId), "Empty uploadId");
+ verify(length >= 0, "Invalid length: " + length);
+ destinationPath();
+ verify(etags != null, "No etag list");
+ validateCollectionClass(etags, String.class);
+ for (String etag : etags) {
+ verify(StringUtils.isNotEmpty(etag), "Empty etag");
+ }
+ if (extraData != null) {
+ validateCollectionClass(extraData.keySet(), String.class);
+ validateCollectionClass(extraData.values(), String.class);
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "DelayedCompleteData{");
+ sb.append("version=").append(version);
+ sb.append(", uri='").append(uri).append('\'');
+ sb.append(", destination='").append(destinationKey).append('\'');
+ sb.append(", uploadId='").append(uploadId).append('\'');
+ sb.append(", created=").append(created);
+ sb.append(", saved=").append(saved);
+ sb.append(", size=").append(length);
+ sb.append(", date='").append(date).append('\'');
+ sb.append(", jobId='").append(jobId).append('\'');
+ sb.append(", taskId='").append(taskId).append('\'');
+ sb.append(", notes='").append(text).append('\'');
+ if (etags != null) {
+ sb.append(", etags=[");
+ sb.append(join(",", etags));
+ sb.append(']');
+ } else {
+ sb.append(", etags=null");
+ }
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public byte[] toBytes() throws IOException {
+ validate();
+ return serializer().toBytes(this);
+ }
+
+ @Override
+ public void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException {
+ serializer().save(fs, path, this, overwrite);
+ }
+
+ /**
+ * Build the destination path of the object.
+ * @return the path
+ * @throws IllegalStateException if the URI is invalid
+ */
+ public Path destinationPath() {
+ Preconditions.checkState(StringUtils.isNotEmpty(uri), "Empty uri");
+ try {
+ return new Path(new URI(uri));
+ } catch (URISyntaxException e) {
+ throw new IllegalStateException("Cannot parse URI " + uri);
+ }
+ }
+
+ /**
+ * Get the number of etags.
+ * @return the size of the etag list.
+ */
+ public int getPartCount() {
+ return etags.size();
+ }
+
+ /**
+ * Iterate over the etags.
+ * @return an iterator.
+ */
+ @Override
+ public Iterator<String> iterator() {
+ return etags.iterator();
+ }
+
+ /** @return version marker. */
+ public int getVersion() {
+ return version;
+ }
+
+ public void setVersion(int version) {
+ this.version = version;
+ }
+
+ /**
+ * This is the filename of the pending file itself.
+ * Used during processing; it's persistent value, if any, is ignored.
+ * @return filename
+ */
+ public String getFilename() {
+ return filename;
+ }
+
+ public void setFilename(String filename) {
+ this.filename = filename;
+ }
+
+ /** @return path URI of the destination. */
+ public String getUri() {
+ return uri;
+ }
+
+ public void setUri(String uri) {
+ this.uri = uri;
+ }
+
+ /** @return ID of the upload. */
+ public String getUploadId() {
+ return uploadId;
+ }
+
+ public void setUploadId(String uploadId) {
+ this.uploadId = uploadId;
+ }
+
+ /** @return destination bucket. */
+ public String getBucket() {
+ return bucket;
+ }
+
+ public void setBucket(String bucket) {
+ this.bucket = bucket;
+ }
+
+ /** @return destination key in the bucket. */
+ public String getDestinationKey() {
+ return destinationKey;
+ }
+
+ public void setDestinationKey(String destinationKey) {
+ this.destinationKey = destinationKey;
+ }
+
+ /**
+ * When was the upload created?
+ * @return timestamp
+ */
+ public long getCreated() {
+ return created;
+ }
+
+ public void setCreated(long created) {
+ this.created = created;
+ }
+
+ /**
+ * When was the upload saved?
+ * @return timestamp
+ */
+ public long getSaved() {
+ return saved;
+ }
+
+ public void setSaved(long saved) {
+ this.saved = saved;
+ }
+
+ /**
+ * Timestamp as date; no expectation of parseability.
+ * @return date string
+ */
+ public String getDate() {
+ return date;
+ }
+
+ public void setDate(String date) {
+ this.date = date;
+ }
+
+ /** @return Job ID, if known. */
+ public String getJobId() {
+ return jobId;
+ }
+
+ public void setJobId(String jobId) {
+ this.jobId = jobId;
+ }
+
+ /** @return Task ID, if known. */
+ public String getTaskId() {
+ return taskId;
+ }
+
+ public void setTaskId(String taskId) {
+ this.taskId = taskId;
+ }
+
+ /**
+ * Arbitrary notes.
+ * @return any notes
+ */
+ public String getText() {
+ return text;
+ }
+
+ public void setText(String text) {
+ this.text = text;
+ }
+
+ /** @return ordered list of etags. */
+ public List<String> getEtags() {
+ return etags;
+ }
+
+ public void setEtags(List<String> etags) {
+ this.etags = etags;
+ }
+
+ /**
+ * Any custom extra data committer subclasses may choose to add.
+ * @return custom data
+ */
+ public Map<String, String> getExtraData() {
+ return extraData;
+ }
+
+ public void setExtraData(Map<String, String> extraData) {
+ this.extraData = extraData;
+ }
+
+ /**
+ * Destination file size.
+ * @return size of destination object
+ */
+ public long getLength() {
+ return length;
+ }
+
+ public void setLength(long length) {
+ this.length = length;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
new file mode 100644
index 0000000..6cf1f1e
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/SuccessData.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.files;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.ValidationFailure;
+import org.apache.hadoop.util.JsonSerialization;
+
+/**
+ * Summary data saved into a {@code _SUCCESS} marker file.
+ *
+ * This provides an easy way to determine which committer was used
+ * to commit work.
+ * <ol>
+ * <li>File length == 0: classic {@code FileOutputCommitter}.</li>
+ * <li>Loadable as {@link SuccessData}:
+ * A s3guard committer with name in in {@link #committer} field.</li>
+ * <li>Not loadable? Something else.</li>
+ * </ol>
+ *
+ * This is an unstable structure intended for diagnostics and testing.
+ * Applications reading this data should use/check the {@link #name} field
+ * to differentiate from any other JSON-based manifest and to identify
+ * changes in the output format.
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class SuccessData extends PersistentCommitData {
+ private static final Logger LOG = LoggerFactory.getLogger(SuccessData.class);
+
+ /**
+ * Serialization ID: {@value}.
+ */
+ private static final long serialVersionUID = 507133045258460084L;
+
+ /**
+ * Name to include in persisted data, so as to differentiate from
+ * any other manifests: {@value}.
+ */
+ public static final String NAME
+ = "org.apache.hadoop.fs.s3a.commit.files.SuccessData/1";
+
+ /**
+ * Name of file; includes version marker.
+ */
+ private String name;
+
+ /** Timestamp of creation. */
+ private long timestamp;
+
+ /** Timestamp as date string; no expectation of parseability. */
+ private String date;
+
+ /**
+ * Host which created the file (implicitly: committed the work).
+ */
+ private String hostname;
+
+ /**
+ * Committer name.
+ */
+ private String committer;
+
+ /**
+ * Description text.
+ */
+ private String description;
+
+ /**
+ * Metrics.
+ */
+ private Map<String, Long> metrics = new HashMap<>();
+
+ /**
+ * Diagnostics information.
+ */
+ private Map<String, String> diagnostics = new HashMap<>();
+
+ /**
+ * Filenames in the commit.
+ */
+ private List<String> filenames = new ArrayList<>(0);
+
+ @Override
+ public void validate() throws ValidationFailure {
+ ValidationFailure.verify(name != null,
+ "Incompatible file format: no 'name' field");
+ ValidationFailure.verify(NAME.equals(name),
+ "Incompatible file format: " + name);
+ }
+
+ @Override
+ public byte[] toBytes() throws IOException {
+ return serializer().toBytes(this);
+ }
+
+ @Override
+ public void save(FileSystem fs, Path path, boolean overwrite)
+ throws IOException {
+ // always set the name field before being saved.
+ name = NAME;
+ serializer().save(fs, path, this, overwrite);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "SuccessData{");
+ sb.append("committer='").append(committer).append('\'');
+ sb.append(", hostname='").append(hostname).append('\'');
+ sb.append(", description='").append(description).append('\'');
+ sb.append(", date='").append(date).append('\'');
+ sb.append(", filenames=[").append(
+ StringUtils.join(filenames, ", "))
+ .append("]");
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Dump the metrics (if any) to a string.
+ * The metrics are sorted for ease of viewing.
+ * @param prefix prefix before every entry
+ * @param middle string between key and value
+ * @param suffix suffix to each entry
+ * @return the dumped string
+ */
+ public String dumpMetrics(String prefix, String middle, String suffix) {
+ return joinMap(metrics, prefix, middle, suffix);
+ }
+
+ /**
+ * Dump the diagnostics (if any) to a string.
+ * @param prefix prefix before every entry
+ * @param middle string between key and value
+ * @param suffix suffix to each entry
+ * @return the dumped string
+ */
+ public String dumpDiagnostics(String prefix, String middle, String suffix) {
+ return joinMap(diagnostics, prefix, middle, suffix);
+ }
+
+ /**
+ * Join any map of string to value into a string, sorting the keys first.
+ * @param map map to join
+ * @param prefix prefix before every entry
+ * @param middle string between key and value
+ * @param suffix suffix to each entry
+ * @return a string for reporting.
+ */
+ protected static String joinMap(Map<String, ?> map,
+ String prefix,
+ String middle, String suffix) {
+ if (map == null) {
+ return "";
+ }
+ List<String> list = new ArrayList<>(map.keySet());
+ Collections.sort(list);
+ StringBuilder sb = new StringBuilder(list.size() * 32);
+ for (String k : list) {
+ sb.append(prefix)
+ .append(k)
+ .append(middle)
+ .append(map.get(k))
+ .append(suffix);
+ }
+ return sb.toString();
+ }
+
+ /**
+ * Load an instance from a file, then validate it.
+ * @param fs filesystem
+ * @param path path
+ * @return the loaded instance
+ * @throws IOException IO failure
+ * @throws ValidationFailure if the data is invalid
+ */
+ public static SuccessData load(FileSystem fs, Path path)
+ throws IOException {
+ LOG.debug("Reading success data from {}", path);
+ SuccessData instance = serializer().load(fs, path);
+ instance.validate();
+ return instance;
+ }
+
+ /**
+ * Get a JSON serializer for this class.
+ * @return a serializer.
+ */
+ private static JsonSerialization<SuccessData> serializer() {
+ return new JsonSerialization<>(SuccessData.class, false, true);
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ /** @return timestamp of creation. */
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public void setTimestamp(long timestamp) {
+ this.timestamp = timestamp;
+ }
+
+ /** @return timestamp as date; no expectation of parseability. */
+ public String getDate() {
+ return date;
+ }
+
+ public void setDate(String date) {
+ this.date = date;
+ }
+
+ /**
+ * @return host which created the file (implicitly: committed the work).
+ */
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * @return committer name.
+ */
+ public String getCommitter() {
+ return committer;
+ }
+
+ public void setCommitter(String committer) {
+ this.committer = committer;
+ }
+
+ /**
+ * @return any description text.
+ */
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ /**
+ * @return any metrics.
+ */
+ public Map<String, Long> getMetrics() {
+ return metrics;
+ }
+
+ public void setMetrics(Map<String, Long> metrics) {
+ this.metrics = metrics;
+ }
+
+ /**
+ * @return a list of filenames in the commit.
+ */
+ public List<String> getFilenames() {
+ return filenames;
+ }
+
+ public void setFilenames(List<String> filenames) {
+ this.filenames = filenames;
+ }
+
+ public Map<String, String> getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(Map<String, String> diagnostics) {
+ this.diagnostics = diagnostics;
+ }
+
+ /**
+ * Add a diagnostics entry.
+ * @param key name
+ * @param value value
+ */
+ public void addDiagnostic(String key, String value) {
+ diagnostics.put(key, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java
new file mode 100644
index 0000000..0d57494
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/files/package-info.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Persistent data formats for the committers.
+ *
+ * All of these formats share a base class of
+ * {@link org.apache.hadoop.fs.s3a.commit.files.PersistentCommitData};
+ * the subclasses record
+ * <ol>
+ * <li>The content of a single pending commit
+ * (used by the Magic committer).</li>
+ * <li>The list of all the files uploaded by a staging committer.</li>
+ * <li>The summary information saved in the {@code _SUCCESS} file.</li>
+ * </ol>
+ *
+ * There are no guarantees of stability between versions; these are internal
+ * structures.
+ *
+ * The {@link org.apache.hadoop.fs.s3a.commit.files.SuccessData} file is
+ * the one visible to callers after a job completes; it is an unstable
+ * manifest intended for testing only.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.commit.files;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
new file mode 100644
index 0000000..cf365c2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicCommitTracker.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.util.List;
+
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.WriteOperationHelper;
+import org.apache.hadoop.fs.s3a.commit.PutTracker;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+
+/**
+ * Put tracker for Magic commits.
+ * <p>Important</p>: must not directly or indirectly import a class which
+ * uses any datatype in hadoop-mapreduce.
+ */
+@InterfaceAudience.Private
+public class MagicCommitTracker extends PutTracker {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ MagicCommitTracker.class);
+
+ private final String originalDestKey;
+ private final String pendingPartKey;
+ private final Path path;
+ private final WriteOperationHelper writer;
+ private final String bucket;
+ private static final byte[] EMPTY = new byte[0];
+
+ /**
+ * Magic commit tracker.
+ * @param path path nominally being written to
+ * @param bucket dest bucket
+ * @param originalDestKey the original key, in the magic directory.
+ * @param destKey key for the destination
+ * @param pendingsetKey key of the pendingset file
+ * @param writer writer instance to use for operations
+ */
+ public MagicCommitTracker(Path path,
+ String bucket,
+ String originalDestKey,
+ String destKey,
+ String pendingsetKey,
+ WriteOperationHelper writer) {
+ super(destKey);
+ this.bucket = bucket;
+ this.path = path;
+ this.originalDestKey = originalDestKey;
+ this.pendingPartKey = pendingsetKey;
+ this.writer = writer;
+ LOG.info("File {} is written as magic file to path {}",
+ path, destKey);
+ }
+
+ /**
+ * Initialize the tracker.
+ * @return true, indicating that the multipart commit must start.
+ * @throws IOException any IO problem.
+ */
+ @Override
+ public boolean initialize() throws IOException {
+ return true;
+ }
+
+ /**
+ * Flag to indicate that output is not visible after the stream
+ * is closed.
+ * @return true
+ */
+ @Override
+ public boolean outputImmediatelyVisible() {
+ return false;
+ }
+
+ /**
+ * Complete operation: generate the final commit data, put it.
+ * @param uploadId Upload ID
+ * @param parts list of parts
+ * @param bytesWritten bytes written
+ * @return false, indicating that the commit must fail.
+ * @throws IOException any IO problem.
+ * @throws IllegalArgumentException bad argument
+ */
+ @Override
+ public boolean aboutToComplete(String uploadId,
+ List<PartETag> parts,
+ long bytesWritten)
+ throws IOException {
+ Preconditions.checkArgument(StringUtils.isNotEmpty(uploadId),
+ "empty/null upload ID: "+ uploadId);
+ Preconditions.checkArgument(parts != null,
+ "No uploaded parts list");
+ Preconditions.checkArgument(!parts.isEmpty(),
+ "No uploaded parts to save");
+ SinglePendingCommit commitData = new SinglePendingCommit();
+ commitData.touch(System.currentTimeMillis());
+ commitData.setDestinationKey(getDestKey());
+ commitData.setBucket(bucket);
+ commitData.setUri(path.toUri().toString());
+ commitData.setUploadId(uploadId);
+ commitData.setText("");
+ commitData.setLength(bytesWritten);
+ commitData.bindCommitData(parts);
+ byte[] bytes = commitData.toBytes();
+ LOG.info("Uncommitted data pending to file {};"
+ + " commit metadata for {} parts in {}. sixe: {} byte(s)",
+ path.toUri(), parts.size(), pendingPartKey, bytesWritten);
+ LOG.debug("Closed MPU to {}, saved commit information to {}; data=:\n{}",
+ path, pendingPartKey, commitData);
+ PutObjectRequest put = writer.createPutObjectRequest(
+ pendingPartKey,
+ new ByteArrayInputStream(bytes),
+ bytes.length);
+ writer.uploadObject(put);
+
+ // now put a 0-byte file with the name of the original under-magic path
+ PutObjectRequest originalDestPut = writer.createPutObjectRequest(
+ originalDestKey,
+ new ByteArrayInputStream(EMPTY),
+ 0);
+ writer.uploadObject(originalDestPut);
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "MagicCommitTracker{");
+ sb.append(", destKey=").append(getDestKey());
+ sb.append(", pendingPartKey='").append(pendingPartKey).append('\'');
+ sb.append(", path=").append(path);
+ sb.append(", writer=").append(writer);
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java
new file mode 100644
index 0000000..c305141
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitter.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
+import org.apache.hadoop.fs.s3a.commit.CommitOperations;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR;
+import org.apache.hadoop.fs.s3a.commit.DurationInfo;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*;
+
+/**
+ * This is a dedicated committer which requires the "magic" directory feature
+ * of the S3A Filesystem to be enabled; it then uses paths for task and job
+ * attempts in magic paths, so as to ensure that the final output goes direct
+ * to the destination directory.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class MagicS3GuardCommitter extends AbstractS3ACommitter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MagicS3GuardCommitter.class);
+
+ /** Name: {@value}. */
+ public static final String NAME = CommitConstants.COMMITTER_NAME_MAGIC;
+
+ /**
+ * Create a task committer.
+ * @param outputPath the job's output path
+ * @param context the task's context
+ * @throws IOException on a failure
+ */
+ public MagicS3GuardCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ setWorkPath(getTaskAttemptPath(context));
+ verifyIsMagicCommitPath(getDestS3AFS(), getWorkPath());
+ LOG.debug("Task attempt {} has work path {}",
+ context.getTaskAttemptID(),
+ getWorkPath());
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ /**
+ * Require magic paths in the FS client.
+ * @return true, always.
+ */
+ @Override
+ protected boolean requiresDelayedCommitOutputInFileSystem() {
+ return true;
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Setup Job %s", jobIdString(context))) {
+ Path jobAttemptPath = getJobAttemptPath(context);
+ getDestinationFS(jobAttemptPath,
+ context.getConfiguration()).mkdirs(jobAttemptPath);
+ }
+ }
+
+ /**
+ * Get the list of pending uploads for this job attempt, by listing
+ * all .pendingset files in the job attempt directory.
+ * @param context job context
+ * @return a list of pending commits.
+ * @throws IOException Any IO failure
+ */
+ protected List<SinglePendingCommit> listPendingUploadsToCommit(
+ JobContext context)
+ throws IOException {
+ FileSystem fs = getDestFS();
+ return loadPendingsetFiles(context, false, fs,
+ listAndFilter(fs, getJobAttemptPath(context), false,
+ CommitOperations.PENDINGSET_FILTER));
+ }
+
+ /**
+ * Delete the magic directory.
+ */
+ public void cleanupStagingDirs() {
+ Path path = magicSubdir(getOutputPath());
+ Invoker.ignoreIOExceptions(LOG, "cleanup magic directory", path.toString(),
+ () -> deleteWithWarning(getDestFS(), path, true));
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Setup Task %s", context.getTaskAttemptID())) {
+ Path taskAttemptPath = getTaskAttemptPath(context);
+ FileSystem fs = taskAttemptPath.getFileSystem(getConf());
+ fs.mkdirs(taskAttemptPath);
+ }
+ }
+
+ /**
+ * Did this task write any files in the work directory?
+ * Probes for a task existing by looking to see if the attempt dir exists.
+ * This adds more HTTP requests to the call. It may be better just to
+ * return true and rely on the commit task doing the work.
+ * @param context the task's context
+ * @return true if the attempt path exists
+ * @throws IOException failure to list the path
+ */
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ Path taskAttemptPath = getTaskAttemptPath(context);
+ try (DurationInfo d = new DurationInfo(LOG,
+ "needsTaskCommit task %s", context.getTaskAttemptID())) {
+ return taskAttemptPath.getFileSystem(
+ context.getConfiguration())
+ .exists(taskAttemptPath);
+ }
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Commit task %s", context.getTaskAttemptID())) {
+ PendingSet commits = innerCommitTask(context);
+ LOG.info("Task {} committed {} files", context.getTaskAttemptID(),
+ commits.size());
+ } catch (IOException e) {
+ getCommitOperations().taskCompleted(false);
+ throw e;
+ } finally {
+ // delete the task attempt so there's no possibility of a second attempt
+ deleteTaskAttemptPathQuietly(context);
+ }
+ getCommitOperations().taskCompleted(true);
+ }
+
+ /**
+ * Inner routine for committing a task.
+ * The list of pending commits is loaded and then saved to the job attempt
+ * dir.
+ * Failure to load any file or save the final file triggers an abort of
+ * all known pending commits.
+ * @param context context
+ * @return the summary file
+ * @throws IOException exception
+ */
+ private PendingSet innerCommitTask(
+ TaskAttemptContext context) throws IOException {
+ Path taskAttemptPath = getTaskAttemptPath(context);
+ // load in all pending commits.
+ CommitOperations actions = getCommitOperations();
+ Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
+ loaded = actions.loadSinglePendingCommits(
+ taskAttemptPath, true);
+ PendingSet pendingSet = loaded.getKey();
+ List<Pair<LocatedFileStatus, IOException>> failures = loaded.getValue();
+ if (!failures.isEmpty()) {
+ // At least one file failed to load
+ // revert all which did; report failure with first exception
+ LOG.error("At least one commit file could not be read: failing");
+ abortPendingUploads(context, pendingSet.getCommits(), true);
+ throw failures.get(0).getValue();
+ }
+ // patch in IDs
+ String jobId = String.valueOf(context.getJobID());
+ String taskId = String.valueOf(context.getTaskAttemptID());
+ for (SinglePendingCommit commit : pendingSet.getCommits()) {
+ commit.setJobId(jobId);
+ commit.setTaskId(taskId);
+ }
+
+ Path jobAttemptPath = getJobAttemptPath(context);
+ TaskAttemptID taskAttemptID = context.getTaskAttemptID();
+ Path taskOutcomePath = new Path(jobAttemptPath,
+ taskAttemptID.getTaskID().toString() +
+ CommitConstants.PENDINGSET_SUFFIX);
+ LOG.info("Saving work of {} to {}", taskAttemptID, taskOutcomePath);
+ try {
+ pendingSet.save(getDestFS(), taskOutcomePath, false);
+ } catch (IOException e) {
+ LOG.warn("Failed to save task commit data to {} ",
+ taskOutcomePath, e);
+ abortPendingUploads(context, pendingSet.getCommits(), true);
+ throw e;
+ }
+ return pendingSet;
+ }
+
+ /**
+ * Abort a task. Attempt load then abort all pending files,
+ * then try to delete the task attempt path.
+ * This method may be called on the job committer, rather than the
+ * task one (such as in the MapReduce AM after a task container failure).
+ * It must extract all paths and state from the passed in context.
+ * @param context task context
+ * @throws IOException if there was some problem querying the path other
+ * than it not actually existing.
+ */
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ Path attemptPath = getTaskAttemptPath(context);
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Abort task %s", context.getTaskAttemptID())) {
+ getCommitOperations().abortAllSinglePendingCommits(attemptPath, true);
+ } finally {
+ deleteQuietly(
+ attemptPath.getFileSystem(context.getConfiguration()),
+ attemptPath, true);
+ }
+ }
+
+ /**
+ * Compute the path where the output of a given job attempt will be placed.
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @return the path to store job attempt data.
+ */
+ protected Path getJobAttemptPath(int appAttemptId) {
+ return getMagicJobAttemptPath(appAttemptId, getOutputPath());
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until
+ * that task is committed.
+ *
+ * @param context the context of the task attempt.
+ * @return the path where a task attempt should be stored.
+ */
+ public Path getTaskAttemptPath(TaskAttemptContext context) {
+ return getMagicTaskAttemptPath(context, getOutputPath());
+ }
+
+ @Override
+ protected Path getBaseTaskAttemptPath(TaskAttemptContext context) {
+ return getBaseMagicTaskAttemptPath(context, getOutputPath());
+ }
+
+ /**
+ * Get a temporary directory for data. When a task is aborted/cleaned
+ * up, the contents of this directory are all deleted.
+ * @param context task context
+ * @return a path for temporary data.
+ */
+ public Path getTempTaskAttemptPath(TaskAttemptContext context) {
+ return CommitUtilsWithMR.getTempTaskAttemptPath(context, getOutputPath());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitterFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitterFactory.java
new file mode 100644
index 0000000..0392491
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/MagicS3GuardCommitterFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitterFactory;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+
+/**
+ * Factory for the Magic committer.
+ */
+public class MagicS3GuardCommitterFactory
+ extends AbstractS3ACommitterFactory {
+
+ /**
+ * Name of this class: {@value}.
+ */
+ public static final String CLASSNAME
+ = "org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitterFactory";
+
+ public PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem,
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new MagicS3GuardCommitter(outputPath, context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/package-info.java
new file mode 100644
index 0000000..56925e9
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/magic/package-info.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This is the "Magic" committer and support.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/package-info.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/package-info.java
new file mode 100644
index 0000000..7647675
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Support for committing the output of analytics jobs directly to S3.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/ConflictResolution.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/ConflictResolution.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/ConflictResolution.java
new file mode 100644
index 0000000..52d3411
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/ConflictResolution.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+/**
+ * Enum of conflict resolution algorithms.
+ */
+public enum ConflictResolution {
+ /** Fail. */
+ FAIL,
+
+ /** Merge new data with existing data. */
+ APPEND,
+
+ /** Overwrite existing data. */
+ REPLACE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
new file mode 100644
index 0000000..03dfed2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.*;
+
+/**
+ * This commits to a directory.
+ * The conflict policy is
+ * <ul>
+ * <li>FAIL: fail the commit</li>
+ * <li>APPEND: add extra data to the destination.</li>
+ * <li>REPLACE: delete the destination directory in the job commit
+ * (i.e. after and only if all tasks have succeeded.</li>
+ * </ul>
+ */
+public class DirectoryStagingCommitter extends StagingCommitter {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ DirectoryStagingCommitter.class);
+
+ /** Name: {@value}. */
+ public static final String NAME = COMMITTER_NAME_DIRECTORY;
+
+ public DirectoryStagingCommitter(Path outputPath, TaskAttemptContext context)
+ throws IOException {
+ super(outputPath, context);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ super.setupJob(context);
+ Path outputPath = getOutputPath();
+ FileSystem fs = getDestFS();
+ if (getConflictResolutionMode(context, fs.getConf())
+ == ConflictResolution.FAIL
+ && fs.exists(outputPath)) {
+ LOG.debug("Failing commit by task attempt {} to write"
+ + " to existing output path {}",
+ context.getJobID(), getOutputPath());
+ throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS);
+ }
+ }
+
+ /**
+ * Pre-commit actions for a job.
+ * Here: look at the conflict resolution mode and choose
+ * an action based on the current policy.
+ * @param context job context
+ * @param pending pending commits
+ * @throws IOException any failure
+ */
+ @Override
+ protected void preCommitJob(JobContext context,
+ List<SinglePendingCommit> pending) throws IOException {
+ Path outputPath = getOutputPath();
+ FileSystem fs = getDestFS();
+ Configuration fsConf = fs.getConf();
+ switch (getConflictResolutionMode(context, fsConf)) {
+ case FAIL:
+ // this was checked in setupJob, but this avoids some cases where
+ // output was created while the job was processing
+ if (fs.exists(outputPath)) {
+ throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS);
+ }
+ break;
+ case APPEND:
+ // do nothing
+ break;
+ case REPLACE:
+ if (fs.delete(outputPath, true /* recursive */)) {
+ LOG.info("{}: removed output path to be replaced: {}",
+ getRole(), outputPath);
+ }
+ break;
+ default:
+ throw new IOException(getRole() + ": unknown conflict resolution mode: "
+ + getConflictResolutionMode(context, fsConf));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitterFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitterFactory.java
new file mode 100644
index 0000000..bfa8914
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitterFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitterFactory;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+
+/**
+ * Factory for the Directory committer.
+ */
+public class DirectoryStagingCommitterFactory
+ extends AbstractS3ACommitterFactory {
+
+ /**
+ * Name of this class: {@value}.
+ */
+ public static final String CLASSNAME
+ = "org.apache.hadoop.fs.s3a.commit.staging"
+ + ".DirectoryStagingCommitterFactory";
+
+ public PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem,
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ return new DirectoryStagingCommitter(outputPath, context);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java
new file mode 100644
index 0000000..bfaf443
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedStagingCommitter.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import com.google.common.collect.Sets;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathExistsException;
+import org.apache.hadoop.fs.s3a.commit.PathCommitException;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.*;
+
+/**
+ * Partitioned committer.
+ * This writes data to specific "partition" subdirectories, applying
+ * conflict resolution on a partition-by-partition basis. The existence
+ * and state of any parallel partitions for which there is no are output
+ * files are not considered in the conflict resolution.
+ *
+ * The conflict policy is
+ * <ul>
+ * <li>FAIL: fail the commit if any of the partitions have data.</li>
+ * <li>APPEND: add extra data to the destination partitions.</li>
+ * <li>REPLACE: delete the destination partition in the job commit
+ * (i.e. after and only if all tasks have succeeded.</li>
+ * </ul>
+ */
+public class PartitionedStagingCommitter extends StagingCommitter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ PartitionedStagingCommitter.class);
+
+ /** Name: {@value}. */
+ public static final String NAME = COMMITTER_NAME_PARTITIONED;
+
+ public PartitionedStagingCommitter(Path outputPath,
+ TaskAttemptContext context)
+ throws IOException {
+ super(outputPath, context);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "PartitionedStagingCommitter{");
+ sb.append(super.toString());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ protected int commitTaskInternal(TaskAttemptContext context,
+ List<? extends FileStatus> taskOutput) throws IOException {
+ Path attemptPath = getTaskAttemptPath(context);
+ Set<String> partitions = Paths.getPartitions(attemptPath, taskOutput);
+
+ // enforce conflict resolution, but only if the mode is FAIL. for APPEND,
+ // it doesn't matter that the partitions are already there, and for REPLACE,
+ // deletion should be done during job commit.
+ FileSystem fs = getDestFS();
+ if (getConflictResolutionMode(context, fs.getConf())
+ == ConflictResolution.FAIL) {
+ for (String partition : partitions) {
+ // getFinalPath adds the UUID to the file name. this needs the parent.
+ Path partitionPath = getFinalPath(partition + "/file",
+ context).getParent();
+ if (fs.exists(partitionPath)) {
+ LOG.debug("Failing commit by task attempt {} to write"
+ + " to existing path {} under {}",
+ context.getTaskAttemptID(), partitionPath, getOutputPath());
+ throw new PathExistsException(partitionPath.toString(),
+ E_DEST_EXISTS);
+ }
+ }
+ }
+ return super.commitTaskInternal(context, taskOutput);
+ }
+
+ /**
+ * Job-side conflict resolution.
+ * The partition path conflict resolution actions are:
+ * <ol>
+ * <li>FAIL: assume checking has taken place earlier; no more checks.</li>
+ * <li>APPEND: allowed.; no need to check.</li>
+ * <li>REPLACE deletes all existing partitions.</li>
+ * </ol>
+ * @param context job context
+ * @param pending the pending operations
+ * @throws IOException any failure
+ */
+ @Override
+ protected void preCommitJob(JobContext context,
+ List<SinglePendingCommit> pending) throws IOException {
+
+ FileSystem fs = getDestFS();
+
+ // enforce conflict resolution
+ Configuration fsConf = fs.getConf();
+ switch (getConflictResolutionMode(context, fsConf)) {
+ case FAIL:
+ // FAIL checking is done on the task side, so this does nothing
+ break;
+ case APPEND:
+ // no check is needed because the output may exist for appending
+ break;
+ case REPLACE:
+ Set<Path> partitions = pending.stream()
+ .map(SinglePendingCommit::destinationPath)
+ .map(Path::getParent)
+ .collect(Collectors.toCollection(Sets::newLinkedHashSet));
+ for (Path partitionPath : partitions) {
+ LOG.debug("{}: removing partition path to be replaced: " +
+ getRole(), partitionPath);
+ fs.delete(partitionPath, true);
+ }
+ break;
+ default:
+ throw new PathCommitException("",
+ getRole() + ": unknown conflict resolution mode: "
+ + getConflictResolutionMode(context, fsConf));
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/49] hadoop git commit: YARN-7483. CapacityScheduler test cases
cleanup post YARN-5881. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7483. CapacityScheduler test cases cleanup post YARN-5881. (Sunil G via wangda)
Change-Id: I9741a6baf5cb7352d05636efb6c0b24790e7589a
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08a71ae6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08a71ae6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08a71ae6
Branch: refs/heads/YARN-5881
Commit: 08a71ae6b183783e9fe705d260cc88949ec89bc4
Parents: c55cc51
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 16 11:35:48 2017 -0800
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../api/records/impl/pb/ResourcePBImpl.java | 11 ++
.../scheduler/capacity/TestLeafQueue.java | 130 +++++++++++++++++--
2 files changed, 131 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08a71ae6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 4f90133..6ebed6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -151,6 +151,17 @@ public class ResourcePBImpl extends Resource {
.newInstance(ResourceInformation.VCORES);
this.setMemorySize(p.getMemory());
this.setVirtualCores(p.getVirtualCores());
+
+ // Update missing resource information on respective index.
+ updateResourceInformationMap(types);
+ }
+
+ private void updateResourceInformationMap(ResourceInformation[] types) {
+ for (int i = 0; i < types.length; i++) {
+ if (resources[i] == null) {
+ resources[i] = ResourceInformation.newInstance(types[i]);
+ }
+ }
}
private static ResourceInformation newDefaultInformation(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08a71ae6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 30bff78..1426e88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -1027,6 +1027,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
// app_0 asks for 3 3-GB containers
@@ -1083,9 +1085,15 @@ public class TestLeafQueue {
a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(12*GB, a.getUsedResources().getMemorySize());
- assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
+ assertEquals(9*GB, a.getUsedResources().getMemorySize());
+ assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize());
+ assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
+
+ assertEquals(4 * GB,
+ app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize());
+
+ assertEquals(1 * GB,
+ app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize());
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -1317,11 +1325,6 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
- ParentQueue root = (ParentQueue) queues
- .get(CapacitySchedulerConfiguration.ROOT);
- root.updateClusterResource(clusterResource,
- new ResourceLimits(clusterResource));
-
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
@@ -1340,6 +1343,11 @@ public class TestLeafQueue {
a.setUserLimit(50);
a.setUserLimitFactor(2);
+ ParentQueue root = (ParentQueue) queues
+ .get(CapacitySchedulerConfiguration.ROOT);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Now, only user_0 should be active since he is the only one with
// outstanding requests
assertEquals("There should only be 1 active user!",
@@ -1368,8 +1376,8 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
-
+ assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
+
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
root.updateClusterResource(clusterResource,
@@ -1405,6 +1413,108 @@ public class TestLeafQueue {
}
@Test
+ public void testUserHeadroomMultiApp() throws Exception {
+ // Mock the queue
+ LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
+ // unset maxCapacity
+ a.setMaxCapacity(1.0f);
+
+ // Users
+ final String user_0 = "user_0";
+ final String user_1 = "user_1";
+
+ // Submit applications
+ final ApplicationAttemptId appAttemptId_0 = TestUtils
+ .getMockApplicationAttemptId(0, 0);
+ FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
+ a.getAbstractUsersManager(), spyRMContext);
+ a.submitApplicationAttempt(app_0, user_0);
+
+ final ApplicationAttemptId appAttemptId_1 = TestUtils
+ .getMockApplicationAttemptId(1, 0);
+ FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
+ a.getAbstractUsersManager(), spyRMContext);
+ a.submitApplicationAttempt(app_1, user_0); // same user
+
+ final ApplicationAttemptId appAttemptId_2 = TestUtils
+ .getMockApplicationAttemptId(2, 0);
+ FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_1, a,
+ a.getAbstractUsersManager(), spyRMContext);
+ a.submitApplicationAttempt(app_2, user_1);
+
+ // Setup some nodes
+ String host_0 = "127.0.0.1";
+ FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
+ 16 * GB);
+ String host_1 = "127.0.0.2";
+ FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
+ 16 * GB);
+
+ Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(
+ app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(),
+ app_1, app_2.getApplicationAttemptId(), app_2);
+ Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(),
+ node_0, node_1.getNodeID(), node_1);
+
+ final int numNodes = 2;
+ Resource clusterResource = Resources.createResource(numNodes * (16 * GB),
+ 1);
+ when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
+ Priority priority = TestUtils.createMockPriority(1);
+
+ app_0.updateResourceRequests(
+ Collections.singletonList(TestUtils.createResourceRequest(
+ ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory)));
+
+ applyCSAssignment(clusterResource,
+ a.assignContainers(clusterResource, node_0,
+ new ResourceLimits(clusterResource),
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+ a, nodes, apps);
+ assertEquals(1 * GB, a.getUsedResources().getMemorySize());
+ assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize());
+ assertEquals(0 * GB, app_1.getCurrentConsumption().getMemorySize());
+ // Now, headroom is the same for all apps for a given user + queue combo
+ // and a change to any app's headroom is reflected for all the user's apps
+ // once those apps are active/have themselves calculated headroom for
+ // allocation at least one time
+ assertEquals(2 * GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(2 * GB, app_1.getHeadroom().getMemorySize());// not yet active
+ assertEquals(3 * GB, app_2.getHeadroom().getMemorySize());// not yet active
+
+ app_1.updateResourceRequests(
+ Collections.singletonList(TestUtils.createResourceRequest(
+ ResourceRequest.ANY, 1 * GB, 2, true, priority, recordFactory)));
+
+ applyCSAssignment(clusterResource,
+ a.assignContainers(clusterResource, node_0,
+ new ResourceLimits(clusterResource),
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY),
+ a, nodes, apps);
+ assertEquals(2 * GB, a.getUsedResources().getMemorySize());
+ assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize());
+ assertEquals(1 * GB, app_1.getCurrentConsumption().getMemorySize());
+ assertEquals(1 * GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(1 * GB, app_1.getHeadroom().getMemorySize());// now active
+ assertEquals(3 * GB, app_2.getHeadroom().getMemorySize());// not yet active
+
+ // Complete container and verify that headroom is updated, for both apps
+ // for the user
+ RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
+ a.completedContainer(clusterResource, app_0, node_0, rmContainer,
+ ContainerStatus.newInstance(rmContainer.getContainerId(),
+ ContainerState.COMPLETE, "",
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL, null, true);
+
+ assertEquals(2 * GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(2 * GB, app_1.getHeadroom().getMemorySize());
+ }
+
+ @Test
public void testSingleQueueWithMultipleUsers() throws Exception {
// Mock the queue
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/49] hadoop git commit: YARN-7538. Fix performance regression
introduced by Capacity Scheduler absolute min/max resource refactoring.
(Sunil G via wangda)
Posted by su...@apache.org.
YARN-7538. Fix performance regression introduced by Capacity Scheduler absolute min/max resource refactoring. (Sunil G via wangda)
Change-Id: Ic9bd7e599c56970fe01cb0e1bba6df7d1f77eb29
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06144679
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06144679
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06144679
Branch: refs/heads/YARN-5881
Commit: 0614467940c0ebfa333892dd0edb3cf44c996877
Parents: 08a71ae
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 23 19:52:05 2017 -0800
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../scheduler/AbstractResourceUsage.java | 68 +++++++++++---------
.../scheduler/capacity/AbstractCSQueue.java | 4 +-
.../scheduler/capacity/CSQueueUtils.java | 8 +--
.../scheduler/capacity/ParentQueue.java | 4 +-
.../capacity/TestCapacityScheduler.java | 2 +-
5 files changed, 46 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06144679/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
index c295323..3203543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicReferenceArray;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -38,17 +39,20 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class AbstractResourceUsage {
protected ReadLock readLock;
protected WriteLock writeLock;
- protected Map<String, UsageByLabel> usages;
+ protected final Map<String, UsageByLabel> usages;
+ private final UsageByLabel noLabelUsages;
// short for no-label :)
- private static final String NL = CommonNodeLabelsManager.NO_LABEL;
public AbstractResourceUsage() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
- usages = new HashMap<String, UsageByLabel>();
- usages.put(NL, new UsageByLabel(NL));
+ usages = new HashMap<>();
+
+ // For default label, avoid map for faster access.
+ noLabelUsages = new UsageByLabel();
+ usages.put(CommonNodeLabelsManager.NO_LABEL, noLabelUsages);
}
// Usage enum here to make implement cleaner
@@ -57,41 +61,40 @@ public class AbstractResourceUsage {
// be written by ordering policies
USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING(
5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE(
- 9), EFF_MAX_RESOURCE(
- 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12);
+ 9), EFF_MAX_RESOURCE(10);
private int idx;
- private ResourceType(int value) {
+ ResourceType(int value) {
this.idx = value;
}
}
public static class UsageByLabel {
// usage by label, contains all UsageType
- private Resource[] resArr;
+ private final AtomicReferenceArray<Resource> resArr;
- public UsageByLabel(String label) {
- resArr = new Resource[ResourceType.values().length];
- for (int i = 0; i < resArr.length; i++) {
- resArr[i] = Resource.newInstance(0, 0);
- };
+ public UsageByLabel() {
+ resArr = new AtomicReferenceArray<>(ResourceType.values().length);
+ for (int i = 0; i < resArr.length(); i++) {
+ resArr.set(i, Resource.newInstance(0, 0));
+ }
}
public Resource getUsed() {
- return resArr[ResourceType.USED.idx];
+ return resArr.get(ResourceType.USED.idx);
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
- sb.append("{used=" + resArr[0] + "%, ");
- sb.append("pending=" + resArr[1] + "%, ");
- sb.append("am_used=" + resArr[2] + "%, ");
- sb.append("reserved=" + resArr[3] + "%}");
- sb.append("min_eff=" + resArr[9] + "%, ");
- sb.append("max_eff=" + resArr[10] + "%}");
- sb.append("min_effup=" + resArr[11] + "%, ");
+ sb.append("{used=" + resArr.get(ResourceType.USED.idx) + ", ");
+ sb.append("pending=" + resArr.get(ResourceType.PENDING.idx) + ", ");
+ sb.append("am_used=" + resArr.get(ResourceType.AMUSED.idx) + ", ");
+ sb.append("reserved=" + resArr.get(ResourceType.RESERVED.idx) + ", ");
+ sb.append("min_eff=" + resArr.get(ResourceType.EFF_MIN_RESOURCE.idx) + ", ");
+ sb.append(
+ "max_eff=" + resArr.get(ResourceType.EFF_MAX_RESOURCE.idx) + "}");
return sb.toString();
}
}
@@ -104,8 +107,8 @@ public class AbstractResourceUsage {
}
protected Resource _get(String label, ResourceType type) {
- if (label == null) {
- label = RMNodeLabelsManager.NO_LABEL;
+ if (label == null || label.equals(RMNodeLabelsManager.NO_LABEL)) {
+ return normalize(noLabelUsages.resArr.get(type.idx));
}
try {
@@ -114,7 +117,7 @@ public class AbstractResourceUsage {
if (null == usage) {
return Resources.none();
}
- return normalize(usage.resArr[type.idx]);
+ return normalize(usage.resArr.get(type.idx));
} finally {
readLock.unlock();
}
@@ -126,7 +129,7 @@ public class AbstractResourceUsage {
Resource allOfType = Resources.createResource(0);
for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
//all usages types are initialized
- Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
+ Resources.addTo(allOfType, usageEntry.getValue().resArr.get(type.idx));
}
return allOfType;
} finally {
@@ -135,11 +138,12 @@ public class AbstractResourceUsage {
}
private UsageByLabel getAndAddIfMissing(String label) {
- if (label == null) {
- label = RMNodeLabelsManager.NO_LABEL;
+ if (label == null || label.equals(RMNodeLabelsManager.NO_LABEL)) {
+ return noLabelUsages;
}
+
if (!usages.containsKey(label)) {
- UsageByLabel u = new UsageByLabel(label);
+ UsageByLabel u = new UsageByLabel();
usages.put(label, u);
return u;
}
@@ -151,7 +155,7 @@ public class AbstractResourceUsage {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.resArr[type.idx] = res;
+ usage.resArr.set(type.idx, res);
} finally {
writeLock.unlock();
}
@@ -161,7 +165,8 @@ public class AbstractResourceUsage {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- Resources.addTo(usage.resArr[type.idx], res);
+ usage.resArr.set(type.idx,
+ Resources.add(usage.resArr.get(type.idx), res));
} finally {
writeLock.unlock();
}
@@ -171,7 +176,8 @@ public class AbstractResourceUsage {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- Resources.subtractFrom(usage.resArr[type.idx], res);
+ usage.resArr.set(type.idx,
+ Resources.subtract(usage.resArr.get(type.idx), res));
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06144679/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 9caf589..140ea5d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -932,7 +932,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.incUsed(nodeLabel, resourceToInc);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- Resources.none(), nodeLabel, this);
+ nodeLabel, this);
if (null != parent) {
parent.incUsedResource(nodeLabel, resourceToInc, null);
}
@@ -948,7 +948,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.decUsed(nodeLabel, resourceToDec);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- Resources.none(), nodeLabel, this);
+ nodeLabel, this);
if (null != parent) {
parent.decUsedResource(nodeLabel, resourceToDec, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06144679/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 81dec80..6daca51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -180,8 +180,8 @@ class CSQueueUtils {
* used resource for all partitions of this queue.
*/
public static void updateUsedCapacity(final ResourceCalculator rc,
- final Resource totalPartitionResource, Resource clusterResource,
- String nodePartition, AbstractCSQueue childQueue) {
+ final Resource totalPartitionResource, String nodePartition,
+ AbstractCSQueue childQueue) {
QueueCapacities queueCapacities = childQueue.getQueueCapacities();
CSQueueMetrics queueMetrics = childQueue.getMetrics();
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
@@ -287,11 +287,11 @@ class CSQueueUtils {
for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
queueResourceUsage.getNodePartitionsSet())) {
updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
- cluster, partition, childQueue);
+ partition, childQueue);
}
} else {
updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
- cluster, nodePartition, childQueue);
+ nodePartition, childQueue);
}
// Update queue metrics w.r.t node labels. In a generic way, we can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06144679/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index a427fb1..c198d13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -739,8 +739,8 @@ public class ParentQueue extends AbstractCSQueue {
child.getQueueResourceUsage().getUsed(nodePartition));
// Get child's max resource
- Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
- nodePartition, minimumAllocation);
+ Resource childConfiguredMaxResource = child
+ .getEffectiveMaxCapacityDown(nodePartition, minimumAllocation);
// Child's limit should be capped by child configured max resource
childLimit =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06144679/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index ebc5174..1edb0da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4306,7 +4306,7 @@ public class TestCapacityScheduler {
null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
- assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
+ assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
cs.getQueue("p2").getUsedResources().getMemorySize());
//Free a container from X1
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
new file mode 100644
index 0000000..55e3e37
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3AFileSystem.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.net.URI;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase;
+import org.apache.hadoop.util.Progressable;
+
+/**
+ * Relays FS calls to the mocked FS, allows for some extra logging with
+ * stack traces to be included, stubbing out other methods
+ * where needed to avoid failures.
+ *
+ * The logging is useful for tracking
+ * down why there are extra calls to a method than a test would expect:
+ * changes in implementation details often trigger such false-positive
+ * test failures.
+ *
+ * This class is in the s3a package so that it has access to methods
+ */
+public class MockS3AFileSystem extends S3AFileSystem {
+ public static final String BUCKET = "bucket-name";
+ public static final URI FS_URI = URI.create("s3a://" + BUCKET + "/");
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(MockS3AFileSystem.class);
+
+ private final S3AFileSystem mock;
+ private final Pair<StagingTestBase.ClientResults,
+ StagingTestBase.ClientErrors> outcome;
+
+ /** Log nothing: {@value}. */
+ public static final int LOG_NONE = 0;
+
+ /** Log the name of the operation any arguments: {@value}. */
+ public static final int LOG_NAME = 1;
+
+ /** Log the entire stack of where operations are called: {@value}. */
+ public static final int LOG_STACK = 2;
+
+ /**
+ * This can be edited to set the log level of events through the
+ * mock FS.
+ */
+ private int logEvents = LOG_NAME;
+ private final S3AInstrumentation instrumentation =
+ new S3AInstrumentation(FS_URI);
+ private Configuration conf;
+
+ public MockS3AFileSystem(S3AFileSystem mock,
+ Pair<StagingTestBase.ClientResults, StagingTestBase.ClientErrors> outcome) {
+ this.mock = mock;
+ this.outcome = outcome;
+ setUri(FS_URI);
+ setBucket(BUCKET);
+ }
+
+ public Pair<StagingTestBase.ClientResults, StagingTestBase.ClientErrors>
+ getOutcome() {
+ return outcome;
+ }
+
+ public int getLogEvents() {
+ return logEvents;
+ }
+
+ public void setLogEvents(int logEvents) {
+ this.logEvents = logEvents;
+ }
+
+ private void event(String format, Object... args) {
+ Throwable ex = null;
+ String s = String.format(format, args);
+ switch (logEvents) {
+ case LOG_STACK:
+ ex = new Exception(s);
+ /* fall through */
+ case LOG_NAME:
+ LOG.info(s, ex);
+ break;
+ case LOG_NONE:
+ default:
+ //nothing
+ }
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return new Path("s3a://" + BUCKET + "/work");
+ }
+
+ @Override
+ public void initialize(URI name, Configuration originalConf)
+ throws IOException {
+ conf = originalConf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public boolean isMagicCommitEnabled() {
+ return true;
+ }
+
+ /**
+ * Make operation to set the s3 client public.
+ * @param client client.
+ */
+ @Override
+ public void setAmazonS3Client(AmazonS3 client) {
+ LOG.debug("Setting S3 client to {}", client);
+ super.setAmazonS3Client(client);
+ }
+
+ @Override
+ public boolean exists(Path f) throws IOException {
+ event("exists(%s)", f);
+ return mock.exists(f);
+ }
+
+ @Override
+ void finishedWrite(String key, long length) {
+
+ }
+
+ @Override
+ public FSDataInputStream open(Path f, int bufferSize) throws IOException {
+ event("open(%s)", f);
+ return mock.open(f, bufferSize);
+ }
+
+ @Override
+ public FSDataOutputStream create(Path f,
+ FsPermission permission,
+ boolean overwrite,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ event("create(%s)", f);
+ return mock.create(f, permission, overwrite, bufferSize, replication,
+ blockSize, progress);
+ }
+
+ @Override
+ public FSDataOutputStream append(Path f,
+ int bufferSize,
+ Progressable progress) throws IOException {
+ return mock.append(f, bufferSize, progress);
+ }
+
+ @Override
+ public boolean rename(Path src, Path dst) throws IOException {
+ event("rename(%s, %s)", src, dst);
+ return mock.rename(src, dst);
+ }
+
+ @Override
+ public boolean delete(Path f, boolean recursive) throws IOException {
+ event("delete(%s, %s)", f, recursive);
+ return mock.delete(f, recursive);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path f)
+ throws IOException {
+ event("listStatus(%s)", f);
+ return mock.listStatus(f);
+ }
+
+ @Override
+ public RemoteIterator<LocatedFileStatus> listFiles(Path f, boolean recursive)
+ throws IOException {
+ event("listFiles(%s, %s)", f, recursive);
+ return new EmptyIterator();
+ }
+
+ @Override
+ public void setWorkingDirectory(Path newDir) {
+ mock.setWorkingDirectory(newDir);
+ }
+
+ @Override
+ public boolean mkdirs(Path f, FsPermission permission) throws IOException {
+ event("mkdirs(%s)", f);
+ return mock.mkdirs(f, permission);
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path f) throws IOException {
+ event("getFileStatus(%s)", f);
+ return mock.getFileStatus(f);
+ }
+
+ @Override
+ public long getDefaultBlockSize(Path f) {
+ return mock.getDefaultBlockSize(f);
+ }
+
+ @Override
+ protected void incrementStatistic(Statistic statistic) {
+ }
+
+ @Override
+ protected void incrementStatistic(Statistic statistic, long count) {
+ }
+
+ @Override
+ protected void incrementGauge(Statistic statistic, long count) {
+ }
+
+ @Override
+ public void incrementReadOperations() {
+ }
+
+ @Override
+ public void incrementWriteOperations() {
+ }
+
+ @Override
+ public void incrementPutStartStatistics(long bytes) {
+ }
+
+ @Override
+ public void incrementPutCompletedStatistics(boolean success, long bytes) {
+ }
+
+ @Override
+ public void incrementPutProgressStatistics(String key, long bytes) {
+ }
+
+ @Override
+ protected void setOptionalMultipartUploadRequestParameters(
+ InitiateMultipartUploadRequest req) {
+// no-op
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public long getDefaultBlockSize() {
+ return mock.getDefaultBlockSize();
+ }
+
+ @Override
+ void deleteObjectAtPath(Path f, String key, boolean isFile)
+ throws AmazonClientException, IOException {
+ deleteObject(key);
+ }
+
+ @Override
+ void maybeCreateFakeParentDirectory(Path path)
+ throws IOException, AmazonClientException {
+ // no-op
+ }
+
+ private static class EmptyIterator implements
+ RemoteIterator<LocatedFileStatus> {
+ @Override
+ public boolean hasNext() throws IOException {
+ return false;
+ }
+
+ @Override
+ public LocatedFileStatus next() throws IOException {
+ return null;
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "MockS3AFileSystem{");
+ sb.append("inner mockFS=").append(mock);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ @Override
+ public S3AInstrumentation.CommitterStatistics newCommitterStatistics() {
+ return instrumentation.newCommitterStatistics();
+ }
+
+ @Override
+ public void operationRetried(Exception ex) {
+ /** no-op */
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
index 4e25380..b746bfe5 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
@@ -21,8 +21,10 @@ package org.apache.hadoop.fs.s3a;
import static org.mockito.Mockito.*;
import java.net.URI;
+import java.util.ArrayList;
import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.MultipartUploadListing;
import com.amazonaws.services.s3.model.Region;
/**
@@ -36,6 +38,12 @@ public class MockS3ClientFactory implements S3ClientFactory {
String bucket = name.getHost();
AmazonS3 s3 = mock(AmazonS3.class);
when(s3.doesBucketExist(bucket)).thenReturn(true);
+ // this listing is used in startup if purging is enabled, so
+ // return a stub value
+ MultipartUploadListing noUploads = new MultipartUploadListing();
+ noUploads.setMultipartUploads(new ArrayList<>(0));
+ when(s3.listMultipartUploads(anyObject()))
+ .thenReturn(noUploads);
when(s3.getBucketLocation(anyString()))
.thenReturn(Region.US_West.toString());
return s3;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
index 2c4f009..7f7802d 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java
@@ -163,4 +163,10 @@ public interface S3ATestConstants {
*/
String CONFIGURATION_TEST_ENDPOINT =
"test.fs.s3a.endpoint";
+
+ /**
+ * Property to set to disable caching.
+ */
+ String FS_S3A_IMPL_DISABLE_CACHE
+ = "fs.s3a.impl.disable.cache";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
index b302e72..773c25a 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestUtils.java
@@ -21,10 +21,13 @@ package org.apache.hadoop.fs.s3a;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
import org.apache.hadoop.fs.s3a.s3guard.DynamoDBClientFactory;
import org.apache.hadoop.fs.s3a.s3guard.DynamoDBLocalClientFactory;
import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
@@ -39,9 +42,13 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.List;
+import java.util.stream.Collectors;
import static org.apache.hadoop.fs.contract.ContractTestUtils.skip;
+import static org.apache.hadoop.fs.s3a.InconsistentAmazonS3Client.*;
import static org.apache.hadoop.fs.s3a.S3ATestConstants.*;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.propagateBucketOptions;
@@ -59,6 +66,7 @@ public final class S3ATestUtils {
* a property has been unset.
*/
public static final String UNSET_PROPERTY = "unset";
+ public static final int PURGE_DELAY_SECONDS = 60 * 60;
/**
* Get S3A FS name.
@@ -120,15 +128,19 @@ public final class S3ATestUtils {
S3AFileSystem fs1 = new S3AFileSystem();
//enable purging in tests
if (purge) {
- conf.setBoolean(PURGE_EXISTING_MULTIPART, true);
- // but a long delay so that parallel multipart tests don't
+ // purge with but a delay so that parallel multipart tests don't
// suddenly start timing out
- conf.setInt(PURGE_EXISTING_MULTIPART_AGE, 30 * 60);
+ enableMultipartPurge(conf, PURGE_DELAY_SECONDS);
}
fs1.initialize(testURI, conf);
return fs1;
}
+ public static void enableMultipartPurge(Configuration conf, int seconds) {
+ conf.setBoolean(PURGE_EXISTING_MULTIPART, true);
+ conf.setInt(PURGE_EXISTING_MULTIPART_AGE, seconds);
+ }
+
/**
* Create a file context for tests.
*
@@ -287,13 +299,13 @@ public final class S3ATestUtils {
* @return the exception, if it is of the expected class
* @throws Exception the exception passed in.
*/
- public static Exception verifyExceptionClass(Class clazz,
+ public static <E extends Throwable> E verifyExceptionClass(Class<E> clazz,
Exception ex)
throws Exception {
if (!(ex.getClass().equals(clazz))) {
throw ex;
}
- return ex;
+ return (E)ex;
}
/**
@@ -302,7 +314,7 @@ public final class S3ATestUtils {
* @param conf configuration to patch
*/
public static void disableFilesystemCaching(Configuration conf) {
- conf.setBoolean("fs.s3a.impl.disable.cache", true);
+ conf.setBoolean(FS_S3A_IMPL_DISABLE_CACHE, true);
}
/**
@@ -732,4 +744,112 @@ public final class S3ATestUtils {
= (S3ABlockOutputStream) out.getWrappedStream();
return blockOutputStream.getStatistics();
}
+
+ /**
+ * Read in a file and convert to an ascii string.
+ * @param fs filesystem
+ * @param path path to read
+ * @return the bytes read and converted to a string
+ * @throws IOException IO problems
+ */
+ public static String read(FileSystem fs,
+ Path path) throws IOException {
+ FileStatus status = fs.getFileStatus(path);
+ try (FSDataInputStream in = fs.open(path)) {
+ byte[] buf = new byte[(int)status.getLen()];
+ in.readFully(0, buf);
+ return new String(buf);
+ }
+ }
+
+ /**
+ * List a directory.
+ * @param fileSystem FS
+ * @param path path
+ * @throws IOException failure.
+ */
+ public static void lsR(FileSystem fileSystem, Path path, boolean recursive)
+ throws Exception {
+ if (path == null) {
+ // surfaces when someone calls getParent() on something at the top
+ // of the path
+ LOG.info("Empty path");
+ return;
+ }
+ S3AUtils.applyLocatedFiles(fileSystem.listFiles(path, recursive),
+ (status) -> LOG.info(" {}", status));
+ }
+
+ /**
+ * Turn on the inconsistent S3A FS client in a configuration,
+ * with 100% probability of inconsistency, default delays.
+ * For this to go live, the paths must include the element
+ * {@link InconsistentAmazonS3Client#DEFAULT_DELAY_KEY_SUBSTRING}.
+ * @param conf configuration to patch
+ * @param delay delay in millis
+ */
+ public static void enableInconsistentS3Client(Configuration conf,
+ long delay) {
+ LOG.info("Enabling inconsistent S3 client");
+ conf.setClass(S3_CLIENT_FACTORY_IMPL, InconsistentS3ClientFactory.class,
+ S3ClientFactory.class);
+ conf.set(FAIL_INJECT_INCONSISTENCY_KEY, DEFAULT_DELAY_KEY_SUBSTRING);
+ conf.setLong(FAIL_INJECT_INCONSISTENCY_MSEC, delay);
+ conf.setFloat(FAIL_INJECT_INCONSISTENCY_PROBABILITY, 0.0f);
+ conf.setFloat(FAIL_INJECT_THROTTLE_PROBABILITY, 0.0f);
+ }
+
+ /**
+ * Is the filesystem using the inconsistent/throttling/unreliable client?
+ * @param fs filesystem
+ * @return true if the filesystem's client is the inconsistent one.
+ */
+ public static boolean isFaultInjecting(S3AFileSystem fs) {
+ return fs.getAmazonS3Client() instanceof InconsistentAmazonS3Client;
+ }
+
+ /**
+ * Skip a test because the client is using fault injection.
+ * This should only be done for those tests which are measuring the cost
+ * of operations or otherwise cannot handle retries.
+ * @param fs filesystem to check
+ */
+ public static void skipDuringFaultInjection(S3AFileSystem fs) {
+ Assume.assumeFalse("Skipping as filesystem has fault injection",
+ isFaultInjecting(fs));
+ }
+
+ /**
+ * Date format used for mapping upload initiation time to human string.
+ */
+ private static final DateFormat LISTING_FORMAT = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss");
+
+ /**
+ * Get a list of all pending uploads under a prefix, one which can be printed.
+ * @param prefix prefix to look under
+ * @return possibly empty list
+ * @throws IOException IO failure.
+ */
+ public static List<String> listMultipartUploads(S3AFileSystem fs,
+ String prefix) throws IOException {
+
+ return fs
+ .listMultipartUploads(prefix).stream()
+ .map(upload -> String.format("Upload to %s with ID %s; initiated %s",
+ upload.getKey(),
+ upload.getUploadId(),
+ LISTING_FORMAT.format(upload.getInitiated())))
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Skip a test if the FS isn't marked as supporting magic commits.
+ * @param fs filesystem
+ */
+ public void assumeMagicCommitEnabled(S3AFileSystem fs) {
+ assume("Magic commit option disabled on " + fs,
+ fs.hasCapability(CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER));
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/StorageStatisticsTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/StorageStatisticsTracker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/StorageStatisticsTracker.java
new file mode 100644
index 0000000..7d17d69
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/StorageStatisticsTracker.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import com.google.common.base.Joiner;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.StorageStatistics;
+
+/**
+ * Class to track storage statistics of a filesystem, generate diffs.
+ */
+public class StorageStatisticsTracker {
+
+ private final FileSystem fs;
+ private Map<String, Long> stats;
+
+ public StorageStatisticsTracker(FileSystem fs) {
+ this.fs = fs;
+ snapshot();
+ }
+
+ public void mark() {
+ stats = snapshot();
+ }
+
+ public Map<String, Long> compare(Map<String, Long> current) {
+ Map<String, Long> diff = new HashMap<>(stats.size());
+ for (Map.Entry<String, Long> entry : stats.entrySet()) {
+ String key = entry.getKey();
+ Long latest = current.get(key);
+ if (latest != null && !latest.equals(entry.getValue())) {
+ diff.put(key, entry.getValue() - latest);
+ }
+ }
+ return diff;
+ }
+
+ public Map<String, Long> compareToCurrent() {
+ return compare(snapshot());
+ }
+
+ public String toString(Map<String, Long> map) {
+ return Joiner.on("\n").withKeyValueSeparator("=").join(map);
+ }
+
+ public Map<String, Long> snapshot() {
+ StatsIterator values = latestValues();
+ Map<String, Long> snapshot = new HashMap<>(
+ stats == null ? 0 : stats.size());
+ for (StorageStatistics.LongStatistic value : values) {
+ snapshot.put(value.getName(), value.getValue());
+ }
+ return snapshot;
+ }
+
+ public StatsIterator latestValues() {
+ return new StatsIterator(fs.getStorageStatistics());
+ }
+
+ /**
+ * Provide an iterator to the stats.
+ */
+ public static class StatsIterator implements
+ Iterable<StorageStatistics.LongStatistic> {
+ private final StorageStatistics statistics;
+
+ public StatsIterator(StorageStatistics statistics) {
+ this.statistics = statistics;
+ }
+
+ @Override
+ public Iterator<StorageStatistics.LongStatistic> iterator() {
+ return statistics.getLongStatistics();
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
new file mode 100644
index 0000000..d29e2df
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestInvoker.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkBaseException;
+import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.net.ConnectTimeoutException;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.Invoker.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.verifyExceptionClass;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.test.LambdaTestUtils.*;
+
+/**
+ * Test the {@link Invoker} code and the associated {@link S3ARetryPolicy}.
+ *
+ * Some of the tests look at how Connection Timeout Exceptions are processed.
+ * Because of how the AWS libraries shade the classes, there have been some
+ * regressions here during development. These tests are intended to verify that
+ * the current match process based on classname works.
+ */
+@SuppressWarnings("ThrowableNotThrown")
+public class TestInvoker extends Assert {
+
+ /** Configuration to use for short retry intervals. */
+ private static final Configuration FAST_RETRY_CONF;
+ private static final ConnectTimeoutException
+ HADOOP_CONNECTION_TIMEOUT_EX = new ConnectTimeoutException("hadoop");
+ private static final Local.ConnectTimeoutException
+ LOCAL_CONNECTION_TIMEOUT_EX
+ = new Local.ConnectTimeoutException("local");
+ private static final org.apache.http.conn.ConnectTimeoutException
+ HTTP_CONNECTION_TIMEOUT_EX
+ = new org.apache.http.conn.ConnectTimeoutException("apache");
+ private static final SocketTimeoutException SOCKET_TIMEOUT_EX
+ = new SocketTimeoutException("socket");
+
+ /**
+ * What retry limit to use.
+ */
+ private static final int ACTIVE_RETRY_LIMIT = RETRY_LIMIT_DEFAULT;
+
+ /**
+ * A retry count guaranteed to be out of range.
+ */
+ private static final int RETRIES_TOO_MANY = ACTIVE_RETRY_LIMIT + 10;
+
+ /**
+ * A count of retry attempts guaranteed to be within the permitted range.
+ */
+ public static final int SAFE_RETRY_COUNT = 5;
+
+ static {
+ FAST_RETRY_CONF = new Configuration();
+ String interval = "10ms";
+ FAST_RETRY_CONF.set(RETRY_INTERVAL, interval);
+ FAST_RETRY_CONF.set(RETRY_THROTTLE_INTERVAL, interval);
+ FAST_RETRY_CONF.setInt(RETRY_LIMIT, ACTIVE_RETRY_LIMIT);
+ FAST_RETRY_CONF.setInt(RETRY_THROTTLE_LIMIT, ACTIVE_RETRY_LIMIT);
+ }
+
+ private static final S3ARetryPolicy RETRY_POLICY =
+ new S3ARetryPolicy(FAST_RETRY_CONF);
+
+ private int retryCount;
+ private Invoker invoker = new Invoker(RETRY_POLICY,
+ (text, e, retries, idempotent) -> retryCount++);
+ private static final AmazonClientException CLIENT_TIMEOUT_EXCEPTION =
+ new AmazonClientException(new Local.ConnectTimeoutException("timeout"));
+ private static final AmazonServiceException BAD_REQUEST = serviceException(
+ AWSBadRequestException.STATUS_CODE,
+ "bad request");
+
+ @Before
+ public void setup() {
+ resetCounters();
+ }
+
+ private static AmazonServiceException serviceException(int code,
+ String text) {
+ AmazonServiceException ex = new AmazonServiceException(text);
+ ex.setStatusCode(code);
+ return ex;
+ }
+
+ private static AmazonS3Exception createS3Exception(int code) {
+ return createS3Exception(code, "", null);
+ }
+
+ private static AmazonS3Exception createS3Exception(int code,
+ String message,
+ Throwable inner) {
+ AmazonS3Exception ex = new AmazonS3Exception(message);
+ ex.setStatusCode(code);
+ ex.initCause(inner);
+ return ex;
+ }
+
+ protected <E extends Throwable> void verifyTranslated(
+ int status,
+ Class<E> expected) throws Exception {
+ verifyTranslated(expected, createS3Exception(status));
+ }
+
+ private static <E extends Throwable> E verifyTranslated(Class<E> clazz,
+ SdkBaseException exception) throws Exception {
+ return verifyExceptionClass(clazz,
+ translateException("test", "/", exception));
+ }
+
+ private void resetCounters() {
+ retryCount = 0;
+ }
+
+ @Test
+ public void test503isThrottled() throws Exception {
+ verifyTranslated(503, AWSServiceThrottledException.class);
+ }
+
+ @Test
+ public void testS3500isStatus500Exception() throws Exception {
+ verifyTranslated(500, AWSStatus500Exception.class);
+ }
+
+ @Test
+ public void test500isStatus500Exception() throws Exception {
+ AmazonServiceException ex = new AmazonServiceException("");
+ ex.setStatusCode(500);
+ verifyTranslated(AWSStatus500Exception.class,
+ ex);
+ }
+
+ @Test(expected = org.apache.hadoop.net.ConnectTimeoutException.class)
+ public void testExtractConnectTimeoutException() throws Throwable {
+ throw extractException("", "",
+ new ExecutionException(
+ new AmazonClientException(LOCAL_CONNECTION_TIMEOUT_EX)));
+ }
+
+ @Test(expected = SocketTimeoutException.class)
+ public void testExtractSocketTimeoutException() throws Throwable {
+ throw extractException("", "",
+ new ExecutionException(
+ new AmazonClientException(SOCKET_TIMEOUT_EX)));
+ }
+
+ /**
+ * Make an assertion about a retry policy.
+ * @param text text for error message
+ * @param policy policy to check against
+ * @param expected expected action
+ * @param ex exception to analyze
+ * @param retries number of times there has already been a retry
+ * @param idempotent whether or not the operation is idempotent
+ * @throws Exception if the retry policy raises it
+ * @throws AssertionError if the returned action was not that expected.
+ */
+ private void assertRetryAction(String text,
+ S3ARetryPolicy policy,
+ RetryPolicy.RetryAction expected,
+ Exception ex,
+ int retries,
+ boolean idempotent) throws Exception {
+ RetryPolicy.RetryAction outcome = policy.shouldRetry(ex, retries, 0,
+ idempotent);
+ if (!expected.action.equals(outcome.action)) {
+ throw new AssertionError(
+ String.format(
+ "%s Expected action %s from shouldRetry(%s, %s, %s), but got"
+ + " %s",
+ text,
+ expected, ex.toString(), retries, idempotent,
+ outcome.action),
+ ex);
+ }
+ }
+
+ @Test
+ public void testRetryThrottled() throws Throwable {
+ S3ARetryPolicy policy = RETRY_POLICY;
+ IOException ex = translateException("GET", "/", newThrottledException());
+
+ assertRetryAction("Expected retry on first throttle",
+ policy, RetryPolicy.RetryAction.RETRY,
+ ex, 0, true);
+ int retries = SAFE_RETRY_COUNT;
+ assertRetryAction("Expected retry on repeated throttle",
+ policy, RetryPolicy.RetryAction.RETRY,
+ ex, retries, true);
+ assertRetryAction("Expected retry on non-idempotent throttle",
+ policy, RetryPolicy.RetryAction.RETRY,
+ ex, retries, false);
+ }
+
+ @Test
+ public void testRetryThrottledDDB() throws Throwable {
+ assertRetryAction("Expected retry on connection timeout",
+ RETRY_POLICY, RetryPolicy.RetryAction.RETRY,
+ new ProvisionedThroughputExceededException("IOPs"), 1, false);
+ }
+
+
+ protected AmazonServiceException newThrottledException() {
+ return serviceException(
+ AWSServiceThrottledException.STATUS_CODE, "throttled");
+ }
+
+ /**
+ * Repeatedly retry until a throttle eventually stops being raised.
+ */
+ @Test
+ public void testRetryOnThrottle() throws Throwable {
+
+ final AtomicInteger counter = new AtomicInteger(0);
+ invoker.retry("test", null, false,
+ () -> {
+ if (counter.incrementAndGet() < 5) {
+ throw newThrottledException();
+ }
+ });
+ }
+
+ /**
+ * Non-idempotent operations fail on anything which isn't a throttle
+ * or connectivity problem.
+ */
+ @Test(expected = AWSBadRequestException.class)
+ public void testNoRetryOfBadRequestNonIdempotent() throws Throwable {
+ invoker.retry("test", null, false,
+ () -> {
+ throw serviceException(400, "bad request");
+ });
+ }
+
+ /**
+ * AWS nested socket problems.
+ */
+ @Test
+ public void testRetryAWSConnectivity() throws Throwable {
+ final AtomicInteger counter = new AtomicInteger(0);
+ invoker.retry("test", null, false,
+ () -> {
+ if (counter.incrementAndGet() < ACTIVE_RETRY_LIMIT) {
+ throw CLIENT_TIMEOUT_EXCEPTION;
+ }
+ });
+ assertEquals(ACTIVE_RETRY_LIMIT, counter.get());
+ }
+
+ /**
+ * Repeatedly retry until eventually a bad request succeeds.
+ */
+ @Test
+ public void testRetryBadRequestIdempotent() throws Throwable {
+ final AtomicInteger counter = new AtomicInteger(0);
+ final int attemptsBeforeSuccess = ACTIVE_RETRY_LIMIT;
+ invoker.retry("test", null, true,
+ () -> {
+ if (counter.incrementAndGet() < attemptsBeforeSuccess) {
+ throw BAD_REQUEST;
+ }
+ });
+ assertEquals(attemptsBeforeSuccess, counter.get());
+ assertEquals("retry count ", attemptsBeforeSuccess - 1, retryCount);
+ }
+
+ @Test
+ public void testConnectionRetryPolicyIdempotent() throws Throwable {
+ assertRetryAction("Expected retry on connection timeout",
+ RETRY_POLICY, RetryPolicy.RetryAction.RETRY,
+ HADOOP_CONNECTION_TIMEOUT_EX, 1, true);
+ assertRetryAction("Expected connection timeout failure",
+ RETRY_POLICY, RetryPolicy.RetryAction.FAIL,
+ HADOOP_CONNECTION_TIMEOUT_EX, RETRIES_TOO_MANY, true);
+ }
+
+ /**
+ * Even on a non-idempotent call, connection failures are considered
+ * retryable.
+ */
+ @Test
+ public void testConnectionRetryPolicyNonIdempotent() throws Throwable {
+ assertRetryAction("Expected retry on connection timeout",
+ RETRY_POLICY, RetryPolicy.RetryAction.RETRY,
+ HADOOP_CONNECTION_TIMEOUT_EX, 1, false);
+ }
+
+ /**
+ * Interrupted IOEs are not retryable.
+ */
+ @Test
+ public void testInterruptedIOExceptionRetry() throws Throwable {
+ assertRetryAction("Expected retry on connection timeout",
+ RETRY_POLICY, RetryPolicy.RetryAction.FAIL,
+ new InterruptedIOException("interrupted"), 1, false);
+ }
+
+ @Test
+ public void testUnshadedConnectionTimeoutExceptionMatching()
+ throws Throwable {
+ // connection timeout exceptions are special, but as AWS shades
+ // theirs, we need to string match them
+ verifyTranslated(ConnectTimeoutException.class,
+ new AmazonClientException(HTTP_CONNECTION_TIMEOUT_EX));
+ }
+
+ @Test
+ public void testShadedConnectionTimeoutExceptionMatching() throws Throwable {
+ // connection timeout exceptions are special, but as AWS shades
+ // theirs, we need to string match them
+ verifyTranslated(ConnectTimeoutException.class,
+ new AmazonClientException(LOCAL_CONNECTION_TIMEOUT_EX));
+ }
+
+ @Test
+ public void testShadedConnectionTimeoutExceptionNotMatching()
+ throws Throwable {
+ InterruptedIOException ex = verifyTranslated(InterruptedIOException.class,
+ new AmazonClientException(new Local.NotAConnectTimeoutException()));
+ if (ex instanceof ConnectTimeoutException) {
+ throw ex;
+ }
+ }
+
+ /**
+ * Test that NPEs aren't retried. Also verify that the
+ * catch counts are incremented, while the retry count isn't.
+ */
+ @Test
+ public void testNPEsNotRetried() throws Throwable {
+ assertRetryAction("Expected NPE trigger failure",
+ RETRY_POLICY, RetryPolicy.RetryAction.FAIL,
+ new NullPointerException("oops"), 1, true);
+ // catch notification didn't see it
+ assertEquals("retry count ", 0, retryCount);
+ }
+
+ /**
+ * Container for the local exceptions, to help keep visible which
+ * specific class of exception.
+ */
+ private static class Local {
+
+ /**
+ * A local exception with a name to match the expected one.
+ */
+ private static class ConnectTimeoutException
+ extends InterruptedIOException {
+ ConnectTimeoutException(String s) {
+ super(s);
+ }
+ }
+
+ /**
+ * A local exception whose name should not match.
+ */
+ private static class NotAConnectTimeoutException
+ extends InterruptedIOException {
+
+ }
+
+ }
+
+ @Test
+ public void testQuietlyVoid() {
+ quietlyEval("", "",
+ () -> {
+ throw HADOOP_CONNECTION_TIMEOUT_EX;
+ });
+ }
+
+ @Test
+ public void testQuietlyEvalReturnValueSuccess() {
+ assertOptionalEquals("quietly", 3,
+ quietlyEval("", "", () -> 3));
+ }
+
+ @Test
+ public void testQuietlyEvalReturnValueFail() {
+ // use a variable so IDEs don't warn of numeric overflows
+ int d = 0;
+ assertOptionalUnset("quietly",
+ quietlyEval("", "", () -> 3 / d));
+ }
+
+ @Test
+ public void testDynamoDBThrottleConversion() throws Throwable {
+ ProvisionedThroughputExceededException exceededException =
+ new ProvisionedThroughputExceededException("iops");
+ AWSServiceThrottledException ddb = verifyTranslated(
+ AWSServiceThrottledException.class, exceededException);
+ assertTrue(isThrottleException(exceededException));
+ assertTrue(isThrottleException(ddb));
+ assertRetryAction("Expected throttling retry",
+ RETRY_POLICY,
+ RetryPolicy.RetryAction.RETRY,
+ ddb, SAFE_RETRY_COUNT, false);
+ // and briefly attempt an operation
+ CatchCallback catcher = new CatchCallback();
+ AtomicBoolean invoked = new AtomicBoolean(false);
+ invoker.retry("test", null, false, catcher,
+ () -> {
+ if (!invoked.getAndSet(true)) {
+ throw exceededException;
+ }
+ });
+ // to verify that the ex was translated by the time it
+ // got to the callback
+ verifyExceptionClass(AWSServiceThrottledException.class,
+ catcher.lastException);
+ }
+
+ /**
+ * Catch the exception and preserve it for later queries.
+ */
+ private static final class CatchCallback implements Retried {
+ private IOException lastException;
+ @Override
+ public void onFailure(String text,
+ IOException exception,
+ int retries,
+ boolean idempotent) {
+ lastException = exception;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
index e647327..39a5e3b 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestListing.java
@@ -32,7 +32,7 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import java.util.Set;
-import static org.apache.hadoop.fs.s3a.Listing.ACCEPT_ALL;
+import static org.apache.hadoop.fs.s3a.S3AUtils.ACCEPT_ALL;
import static org.apache.hadoop.fs.s3a.Listing.ProvidedFileStatusIterator;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
index e548ac2..9b86595 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AExceptionTranslation.java
@@ -27,6 +27,7 @@ import java.io.EOFException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
import java.nio.file.AccessDeniedException;
import java.util.Collections;
import java.util.Map;
@@ -38,58 +39,87 @@ import com.amazonaws.services.s3.model.AmazonS3Exception;
import org.junit.Test;
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+
/**
- * Unit test suite covering translation of AWS SDK exceptions to S3A exceptions.
+ * Unit test suite covering translation of AWS SDK exceptions to S3A exceptions,
+ * and retry/recovery policies.
*/
+@SuppressWarnings("ThrowableNotThrown")
public class TestS3AExceptionTranslation {
+ private static final org.apache.http.conn.ConnectTimeoutException
+ HTTP_CONNECTION_TIMEOUT_EX
+ = new org.apache.http.conn.ConnectTimeoutException("apache");
+ private static final SocketTimeoutException SOCKET_TIMEOUT_EX
+ = new SocketTimeoutException("socket");
+
@Test
public void test301ContainsEndpoint() throws Exception {
- AmazonS3Exception s3Exception = createS3Exception("wrong endpoint", 301,
+ String bucket = "bucket.s3-us-west-2.amazonaws.com";
+ int sc301 = 301;
+ AmazonS3Exception s3Exception = createS3Exception("wrong endpoint", sc301,
Collections.singletonMap(S3AUtils.ENDPOINT_KEY,
- "bucket.s3-us-west-2.amazonaws.com"));
- AWSS3IOException ex = (AWSS3IOException)verifyTranslated(
- AWSS3IOException.class, s3Exception);
- assertEquals(301, ex.getStatusCode());
+ bucket));
+ AWSRedirectException ex = verifyTranslated(
+ AWSRedirectException.class, s3Exception);
+ assertStatusCode(sc301, ex);
assertNotNull(ex.getMessage());
- assertTrue(ex.getMessage().contains("bucket.s3-us-west-2.amazonaws.com"));
- assertTrue(ex.getMessage().contains(ENDPOINT));
+
+ assertContained(ex.getMessage(), bucket);
+ assertContained(ex.getMessage(), ENDPOINT);
+ assertExceptionContains(ENDPOINT, ex, "endpoint");
+ assertExceptionContains(bucket, ex, "bucket name");
+ }
+
+ protected void assertContained(String text, String contained) {
+ assertTrue("string \""+ contained + "\" not found in \"" + text + "\"",
+ text != null && text.contains(contained));
+ }
+
+ protected <E extends Throwable> void verifyTranslated(
+ int status,
+ Class<E> expected) throws Exception {
+ verifyTranslated(expected, createS3Exception(status));
+ }
+
+ @Test
+ public void test400isBad() throws Exception {
+ verifyTranslated(400, AWSBadRequestException.class);
}
@Test
public void test401isNotPermittedFound() throws Exception {
- verifyTranslated(AccessDeniedException.class,
- createS3Exception(401));
+ verifyTranslated(401, AccessDeniedException.class);
}
@Test
public void test403isNotPermittedFound() throws Exception {
- verifyTranslated(AccessDeniedException.class,
- createS3Exception(403));
+ verifyTranslated(403, AccessDeniedException.class);
}
@Test
public void test404isNotFound() throws Exception {
- verifyTranslated(FileNotFoundException.class, createS3Exception(404));
+ verifyTranslated(404, FileNotFoundException.class);
}
@Test
public void test410isNotFound() throws Exception {
- verifyTranslated(FileNotFoundException.class, createS3Exception(410));
+ verifyTranslated(410, FileNotFoundException.class);
}
@Test
public void test416isEOF() throws Exception {
- verifyTranslated(EOFException.class, createS3Exception(416));
+ verifyTranslated(416, EOFException.class);
}
@Test
public void testGenericS3Exception() throws Exception {
// S3 exception of no known type
- AWSS3IOException ex = (AWSS3IOException)verifyTranslated(
+ AWSS3IOException ex = verifyTranslated(
AWSS3IOException.class,
createS3Exception(451));
- assertEquals(451, ex.getStatusCode());
+ assertStatusCode(451, ex);
}
@Test
@@ -97,10 +127,19 @@ public class TestS3AExceptionTranslation {
// service exception of no known type
AmazonServiceException ase = new AmazonServiceException("unwind");
ase.setStatusCode(500);
- AWSServiceIOException ex = (AWSServiceIOException)verifyTranslated(
- AWSServiceIOException.class,
+ AWSServiceIOException ex = verifyTranslated(
+ AWSStatus500Exception.class,
ase);
- assertEquals(500, ex.getStatusCode());
+ assertStatusCode(500, ex);
+ }
+
+ protected void assertStatusCode(int expected, AWSServiceIOException ex) {
+ assertNotNull("Null exception", ex);
+ if (expected != ex.getStatusCode()) {
+ throw new AssertionError("Expected status code " + expected
+ + "but got " + ex.getStatusCode(),
+ ex);
+ }
}
@Test
@@ -122,7 +161,7 @@ public class TestS3AExceptionTranslation {
return source;
}
- private static Exception verifyTranslated(Class clazz,
+ private static <E extends Throwable> E verifyTranslated(Class<E> clazz,
AmazonClientException exception) throws Exception {
return verifyExceptionClass(clazz,
translateException("test", "/", exception));
@@ -130,7 +169,8 @@ public class TestS3AExceptionTranslation {
private void assertContainsInterrupted(boolean expected, Throwable thrown)
throws Throwable {
- if (containsInterruptedException(thrown) != expected) {
+ boolean wasInterrupted = containsInterruptedException(thrown) != null;
+ if (wasInterrupted != expected) {
throw thrown;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
new file mode 100644
index 0000000..267d4df
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractCommitITest.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import com.amazonaws.services.s3.AmazonS3;
+import org.junit.Assert;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.AbstractS3ATestBase;
+import org.apache.hadoop.fs.s3a.InconsistentAmazonS3Client;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.WriteOperationHelper;
+import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/**
+ * Base test suite for committer operations.
+ *
+ * By default, these tests enable the inconsistent committer, with
+ * a delay of {@link #CONSISTENCY_DELAY}; they may also have throttling
+ * enabled/disabled.
+ *
+ * <b>Important:</b> all filesystem probes will have to wait for
+ * the FS inconsistency delays and handle things like throttle exceptions,
+ * or disable throttling and fault injection before the probe.
+ *
+ */
+public abstract class AbstractCommitITest extends AbstractS3ATestBase {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractCommitITest.class);
+
+ protected static final int CONSISTENCY_DELAY = 500;
+ protected static final int CONSISTENCY_PROBE_INTERVAL = 500;
+ protected static final int CONSISTENCY_WAIT = CONSISTENCY_DELAY * 2;
+
+ private InconsistentAmazonS3Client inconsistentClient;
+
+ /**
+ * Should the inconsistent S3A client be used?
+ * Default value: true.
+ * @return true for inconsistent listing
+ */
+ public boolean useInconsistentClient() {
+ return true;
+ }
+
+ /**
+ * switch to an inconsistent path if in inconsistent mode.
+ * {@inheritDoc}
+ */
+ @Override
+ protected Path path(String filepath) throws IOException {
+ return useInconsistentClient() ?
+ super.path(InconsistentAmazonS3Client.DEFAULT_DELAY_KEY_SUBSTRING
+ + "/" + filepath)
+ : super.path(filepath);
+ }
+
+ /**
+ * Creates a configuration for commit operations: commit is enabled in the FS
+ * and output is multipart to on-heap arrays.
+ * @return a configuration to use when creating an FS.
+ */
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ conf.setLong(MIN_MULTIPART_THRESHOLD, MULTIPART_MIN_SIZE);
+ conf.setInt(MULTIPART_SIZE, MULTIPART_MIN_SIZE);
+ conf.set(FAST_UPLOAD_BUFFER, FAST_UPLOAD_BUFFER_ARRAY);
+ if (useInconsistentClient()) {
+ enableInconsistentS3Client(conf, CONSISTENCY_DELAY);
+ }
+ return conf;
+ }
+
+ /**
+ * Get the log; can be overridden for test case log.
+ * @return a log.
+ */
+ public Logger log() {
+ return LOG;
+ }
+
+ /***
+ * Bind to the named committer.
+ *
+ * @param conf configuration
+ * @param factory factory name
+ * @param committerName committer; set if non null/empty
+ */
+ protected void bindCommitter(Configuration conf, String factory,
+ String committerName) {
+ conf.set(S3A_COMMITTER_FACTORY_KEY, factory);
+ if (StringUtils.isNotEmpty(committerName)) {
+ conf.set(FS_S3A_COMMITTER_NAME, committerName);
+ }
+ }
+
+ /**
+ * Clean up a directory.
+ * Waits for consistency if needed
+ * @param dir directory
+ * @param conf configuration
+ * @throws IOException failure
+ */
+ public void rmdir(Path dir, Configuration conf) throws IOException {
+ if (dir != null) {
+ describe("deleting %s", dir);
+ FileSystem fs = dir.getFileSystem(conf);
+ waitForConsistency();
+ fs.delete(dir, true);
+ waitForConsistency();
+ }
+ }
+
+ /**
+ * Setup will use inconsistent client if {@link #useInconsistentClient()}
+ * is true.
+ * @throws Exception failure.
+ */
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ if (useInconsistentClient()) {
+ AmazonS3 client = getFileSystem()
+ .getAmazonS3ClientForTesting("fault injection");
+ Assert.assertTrue(
+ "AWS client is not inconsistent, even though the test requirees it "
+ + client,
+ client instanceof InconsistentAmazonS3Client);
+ inconsistentClient = (InconsistentAmazonS3Client) client;
+ }
+ }
+
+ /**
+ * Teardown waits for the consistency delay and resets failure count, so
+ * FS is stable, before the superclass teardown is called. This
+ * should clean things up better.
+ * @throws Exception failure.
+ */
+ @Override
+ public void teardown() throws Exception {
+ waitForConsistency();
+ // make sure there are no failures any more
+ resetFailures();
+ super.teardown();
+ }
+
+ /**
+ * Wait a multiple of the inconsistency delay for things to stabilize;
+ * no-op if the consistent client is used.
+ * @throws InterruptedIOException if the sleep is interrupted
+ */
+ protected void waitForConsistency() throws InterruptedIOException {
+ if (useInconsistentClient() && inconsistentClient != null) {
+ try {
+ Thread.sleep(2* inconsistentClient.getDelayKeyMsec());
+ } catch (InterruptedException e) {
+ throw (InterruptedIOException)
+ (new InterruptedIOException("while waiting for consistency: " + e)
+ .initCause(e));
+ }
+ }
+ }
+
+ /**
+ * Set the throttling factor on requests.
+ * @param p probability of a throttling occurring: 0-1.0
+ */
+ protected void setThrottling(float p) {
+ inconsistentClient.setThrottleProbability(p);
+ }
+
+ /**
+ * Set the throttling factor on requests and number of calls to throttle.
+ * @param p probability of a throttling occurring: 0-1.0
+ * @param limit limit to number of calls which fail
+ */
+ protected void setThrottling(float p, int limit) {
+ inconsistentClient.setThrottleProbability(p);
+ setFailureLimit(limit);
+ }
+
+ /**
+ * Turn off throttling.
+ */
+ protected void resetFailures() {
+ if (inconsistentClient != null) {
+ setThrottling(0, 0);
+ }
+ }
+
+ /**
+ * Set failure limit.
+ * @param limit limit to number of calls which fail
+ */
+ private void setFailureLimit(int limit) {
+ inconsistentClient.setFailureLimit(limit);
+ }
+
+ /**
+ * Abort all multipart uploads under a path.
+ * @param path path for uploads to abort; may be null
+ * @return a count of aborts
+ * @throws IOException trouble.
+ */
+ protected int abortMultipartUploadsUnderPath(Path path) throws IOException {
+ S3AFileSystem fs = getFileSystem();
+ if (fs != null && path != null) {
+ String key = fs.pathToKey(path);
+ WriteOperationHelper writeOps = fs.createWriteOperationHelper();
+ int count = writeOps.abortMultipartUploadsUnderPath(key);
+ if (count > 0) {
+ log().info("Multipart uploads deleted: {}", count);
+ }
+ return count;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Assert that there *are* pending MPUs.
+ * @param path path to look under
+ * @throws IOException IO failure
+ */
+ protected void assertMultipartUploadsPending(Path path) throws IOException {
+ assertTrue("No multipart uploads in progress under " + path,
+ countMultipartUploads(path) > 0);
+ }
+
+ /**
+ * Assert that there *are no* pending MPUs; assertion failure will include
+ * the list of pending writes.
+ * @param path path to look under
+ * @throws IOException IO failure
+ */
+ protected void assertNoMultipartUploadsPending(Path path) throws IOException {
+ List<String> uploads = listMultipartUploads(getFileSystem(),
+ pathToPrefix(path));
+ if (!uploads.isEmpty()) {
+ String result = uploads.stream().collect(Collectors.joining("\n"));
+ fail("Multipart uploads in progress under " + path + " \n" + result);
+ }
+ }
+
+ /**
+ * Count the number of MPUs under a path.
+ * @param path path to scan
+ * @return count
+ * @throws IOException IO failure
+ */
+ protected int countMultipartUploads(Path path) throws IOException {
+ return countMultipartUploads(pathToPrefix(path));
+ }
+
+ /**
+ * Count the number of MPUs under a prefix; also logs them.
+ * @param prefix prefix to scan
+ * @return count
+ * @throws IOException IO failure
+ */
+ protected int countMultipartUploads(String prefix) throws IOException {
+ return listMultipartUploads(getFileSystem(), prefix).size();
+ }
+
+ /**
+ * Map from a path to a prefix.
+ * @param path path
+ * @return the key
+ */
+ private String pathToPrefix(Path path) {
+ return path == null ? "" :
+ getFileSystem().pathToKey(path);
+ }
+
+ /**
+ * Verify that the specified dir has the {@code _SUCCESS} marker
+ * and that it can be loaded.
+ * The contents will be logged and returned.
+ * @param dir directory to scan
+ * @return the loaded success data
+ * @throws IOException IO Failure
+ */
+ protected SuccessData verifySuccessMarker(Path dir) throws IOException {
+ assertPathExists("Success marker",
+ new Path(dir, _SUCCESS));
+ SuccessData successData = loadSuccessMarker(dir);
+ log().info("Success data {}", successData.toString());
+ log().info("Metrics\n{}",
+ successData.dumpMetrics(" ", " = ", "\n"));
+ log().info("Diagnostics\n{}",
+ successData.dumpDiagnostics(" ", " = ", "\n"));
+ return successData;
+ }
+
+ /**
+ * Load the success marker and return the data inside it.
+ * @param dir directory containing the marker
+ * @return the loaded data
+ * @throws IOException on any failure to load or validate the data
+ */
+ protected SuccessData loadSuccessMarker(Path dir) throws IOException {
+ return SuccessData.load(getFileSystem(), new Path(dir, _SUCCESS));
+ }
+
+ /**
+ * Read a UTF-8 file.
+ * @param path path to read
+ * @return string value
+ * @throws IOException IO failure
+ */
+ protected String readFile(Path path) throws IOException {
+ return ContractTestUtils.readUTF8(getFileSystem(), path, -1);
+ }
+
+ /**
+ * Assert that the given dir does not have the {@code _SUCCESS} marker.
+ * @param dir dir to scan
+ * @throws IOException IO Failure
+ */
+ protected void assertSuccessMarkerDoesNotExist(Path dir) throws IOException {
+ assertPathDoesNotExist("Success marker",
+ new Path(dir, _SUCCESS));
+ }
+
+ /**
+ * Closeable which can be used to safely close writers in
+ * a try-with-resources block..
+ */
+ protected static class CloseWriter implements AutoCloseable{
+
+ private final RecordWriter writer;
+ private final TaskAttemptContext context;
+
+ public CloseWriter(RecordWriter writer,
+ TaskAttemptContext context) {
+ this.writer = writer;
+ this.context = context;
+ }
+
+ @Override
+ public void close() {
+ try {
+ writer.close(context);
+ } catch (IOException | InterruptedException e) {
+ LOG.error("When closing {} on context {}",
+ writer, context, e);
+
+ }
+ }
+ }
+
+ /**
+ * Create a task attempt for a Job. This is based on the code
+ * run in the MR AM, creating a task (0) for the job, then a task
+ * attempt (0).
+ * @param jobId job ID
+ * @param jContext job context
+ * @return the task attempt.
+ */
+ public static TaskAttemptContext taskAttemptForJob(JobId jobId,
+ JobContext jContext) {
+ org.apache.hadoop.mapreduce.v2.api.records.TaskId taskID =
+ MRBuilderUtils.newTaskId(jobId, 0,
+ org.apache.hadoop.mapreduce.v2.api.records.TaskType.MAP);
+ org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
+ MRBuilderUtils.newTaskAttemptId(taskID, 0);
+ return new TaskAttemptContextImpl(
+ jContext.getConfiguration(),
+ TypeConverter.fromYarn(attemptID));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitMRJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitMRJob.java
new file mode 100644
index 0000000..13dfd83
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/AbstractITCommitMRJob.java
@@ -0,0 +1,324 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.Sets;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
+import org.apache.hadoop.service.ServiceOperations;
+
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.*;
+
+/** Full integration test of an MR job. */
+public abstract class AbstractITCommitMRJob extends AbstractCommitITest {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractITCommitMRJob.class);
+
+ private static final int TEST_FILE_COUNT = 2;
+ private static final int SCALE_TEST_FILE_COUNT = 20;
+
+ private static MiniDFSClusterService hdfs;
+ private static MiniMRYarnCluster yarn = null;
+ private static JobConf conf = null;
+ private boolean uniqueFilenames = false;
+ private boolean scaleTest;
+
+ protected static FileSystem getDFS() {
+ return hdfs.getClusterFS();
+ }
+
+ @BeforeClass
+ public static void setupClusters() throws IOException {
+ // the HDFS and YARN clusters share the same configuration, so
+ // the HDFS cluster binding is implicitly propagated to YARN
+ conf = new JobConf();
+ conf.setBoolean(JHAdminConfig.MR_HISTORY_CLEANER_ENABLE, false);
+ conf.setLong(CommonConfigurationKeys.FS_DU_INTERVAL_KEY, Long.MAX_VALUE);
+
+ hdfs = new MiniDFSClusterService();
+ hdfs.init(conf);
+ hdfs.start();
+ yarn = new MiniMRYarnCluster("ITCommitMRJob", 2);
+ yarn.init(conf);
+ yarn.start();
+ }
+
+ @SuppressWarnings("ThrowableNotThrown")
+ @AfterClass
+ public static void teardownClusters() throws IOException {
+ conf = null;
+ ServiceOperations.stopQuietly(yarn);
+ ServiceOperations.stopQuietly(hdfs);
+ hdfs = null;
+ yarn = null;
+ }
+
+ public static MiniDFSCluster getHdfs() {
+ return hdfs.getCluster();
+ }
+
+ public static FileSystem getLocalFS() {
+ return hdfs.getLocalFS();
+ }
+
+ @Rule
+ public final TemporaryFolder temp = new TemporaryFolder();
+
+ /**
+ * The name of the committer as returned by
+ * {@link AbstractS3ACommitter#getName()} and used for committer construction.
+ */
+ protected abstract String committerName();
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ scaleTest = getTestPropertyBool(
+ getConfiguration(),
+ KEY_SCALE_TESTS_ENABLED,
+ DEFAULT_SCALE_TESTS_ENABLED);
+ }
+
+ @Override
+ protected int getTestTimeoutMillis() {
+ return SCALE_TEST_TIMEOUT_SECONDS * 1000;
+ }
+
+ @Test
+ public void testMRJob() throws Exception {
+ S3AFileSystem fs = getFileSystem();
+ // final dest is in S3A
+ Path outputPath = path("testMRJob");
+
+ String commitUUID = UUID.randomUUID().toString();
+ String suffix = uniqueFilenames ? ("-" + commitUUID) : "";
+ int numFiles = getTestFileCount();
+ List<String> expectedFiles = new ArrayList<>(numFiles);
+ Set<String> expectedKeys = Sets.newHashSet();
+ for (int i = 0; i < numFiles; i += 1) {
+ File file = temp.newFile(String.valueOf(i) + ".text");
+ try (FileOutputStream out = new FileOutputStream(file)) {
+ out.write(("file " + i).getBytes(StandardCharsets.UTF_8));
+ }
+ String filename = String.format("part-m-%05d%s", i, suffix);
+ Path path = new Path(outputPath, filename);
+ expectedFiles.add(path.toString());
+ expectedKeys.add("/" + fs.pathToKey(path));
+ }
+ Collections.sort(expectedFiles);
+
+ Job mrJob = Job.getInstance(yarn.getConfig(), "test-committer-job");
+ JobConf jobConf = (JobConf) mrJob.getConfiguration();
+ jobConf.setBoolean(FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES,
+ uniqueFilenames);
+
+
+ bindCommitter(jobConf,
+ CommitConstants.S3A_COMMITTER_FACTORY,
+ committerName());
+ // pass down the scale test flag
+ jobConf.setBoolean(KEY_SCALE_TESTS_ENABLED, scaleTest);
+
+ mrJob.setOutputFormatClass(LoggingTextOutputFormat.class);
+ FileOutputFormat.setOutputPath(mrJob, outputPath);
+
+ File mockResultsFile = temp.newFile("committer.bin");
+ mockResultsFile.delete();
+ String committerPath = "file:" + mockResultsFile;
+ jobConf.set("mock-results-file", committerPath);
+ jobConf.set(FS_S3A_COMMITTER_STAGING_UUID, commitUUID);
+
+ mrJob.setInputFormatClass(TextInputFormat.class);
+ FileInputFormat.addInputPath(mrJob, new Path(temp.getRoot().toURI()));
+
+ mrJob.setMapperClass(MapClass.class);
+ mrJob.setNumReduceTasks(0);
+
+ // an attempt to set up log4j properly, which clearly doesn't work
+ URL log4j = getClass().getClassLoader().getResource("log4j.properties");
+ if (log4j != null && log4j.getProtocol().equals("file")) {
+ Path log4jPath = new Path(log4j.toURI());
+ LOG.debug("Using log4j path {}", log4jPath);
+ mrJob.addFileToClassPath(log4jPath);
+ String sysprops = String.format("-Xmx256m -Dlog4j.configuration=%s",
+ log4j);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, sysprops);
+ jobConf.set("yarn.app.mapreduce.am.command-opts", sysprops);
+ }
+
+ applyCustomConfigOptions(jobConf);
+ // fail fast if anything goes wrong
+ mrJob.setMaxMapAttempts(1);
+
+ mrJob.submit();
+ try (DurationInfo d = new DurationInfo(LOG, "Job Execution")) {
+ boolean succeeded = mrJob.waitForCompletion(true);
+ assertTrue("MR job failed", succeeded);
+ }
+
+ waitForConsistency();
+ assertIsDirectory(outputPath);
+ FileStatus[] results = fs.listStatus(outputPath,
+ S3AUtils.HIDDEN_FILE_FILTER);
+ int fileCount = results.length;
+ List<String> actualFiles = new ArrayList<>(fileCount);
+ assertTrue("No files in output directory", fileCount != 0);
+ LOG.info("Found {} files", fileCount);
+ for (FileStatus result : results) {
+ LOG.debug("result: {}", result);
+ actualFiles.add(result.getPath().toString());
+ }
+ Collections.sort(actualFiles);
+
+ // load in the success data marker: this guarantees that a s3guard
+ // committer was used
+ Path success = new Path(outputPath, _SUCCESS);
+ FileStatus status = fs.getFileStatus(success);
+ assertTrue("0 byte success file - not a s3guard committer " + success,
+ status.getLen() > 0);
+ SuccessData successData = SuccessData.load(fs, success);
+ String commitDetails = successData.toString();
+ LOG.info("Committer name " + committerName() + "\n{}",
+ commitDetails);
+ LOG.info("Committer statistics: \n{}",
+ successData.dumpMetrics(" ", " = ", "\n"));
+ LOG.info("Diagnostics\n{}",
+ successData.dumpDiagnostics(" ", " = ", "\n"));
+ assertEquals("Wrong committer in " + commitDetails,
+ committerName(), successData.getCommitter());
+ List<String> successFiles = successData.getFilenames();
+ assertTrue("No filenames in " + commitDetails,
+ !successFiles.isEmpty());
+
+ assertEquals("Should commit the expected files",
+ expectedFiles, actualFiles);
+
+ Set<String> summaryKeys = Sets.newHashSet();
+ summaryKeys.addAll(successFiles);
+ assertEquals("Summary keyset doesn't list the the expected paths "
+ + commitDetails, expectedKeys, summaryKeys);
+ assertPathDoesNotExist("temporary dir",
+ new Path(outputPath, CommitConstants.TEMPORARY));
+ customPostExecutionValidation(outputPath, successData);
+ }
+
+ /**
+ * Get the file count for the test.
+ * @return the number of mappers to create.
+ */
+ public int getTestFileCount() {
+ return scaleTest ? SCALE_TEST_FILE_COUNT : TEST_FILE_COUNT;
+ }
+
+ /**
+ * Override point to let implementations tune the MR Job conf.
+ * @param c configuration
+ */
+ protected void applyCustomConfigOptions(Configuration c) {
+
+ }
+
+ /**
+ * Override point for any committer specific validation operations;
+ * called after the base assertions have all passed.
+ * @param destPath destination of work
+ * @param successData loaded success data
+ * @throws Exception failure
+ */
+ protected void customPostExecutionValidation(Path destPath,
+ SuccessData successData)
+ throws Exception {
+
+ }
+
+ /**
+ * Test Mapper.
+ * This is executed in separate process, and must not make any assumptions
+ * about external state.
+ */
+ public static class MapClass
+ extends Mapper<LongWritable, Text, LongWritable, Text> {
+
+ private int operations;
+ private String id = "";
+ private LongWritable l = new LongWritable();
+ private Text t = new Text();
+
+ @Override
+ protected void setup(Context context)
+ throws IOException, InterruptedException {
+ super.setup(context);
+ // force in Log4J logging
+ org.apache.log4j.BasicConfigurator.configure();
+ boolean scaleMap = context.getConfiguration()
+ .getBoolean(KEY_SCALE_TESTS_ENABLED, false);
+ operations = scaleMap ? 1000 : 10;
+ id = context.getTaskAttemptID().toString();
+ }
+
+ @Override
+ protected void map(LongWritable key, Text value, Context context)
+ throws IOException, InterruptedException {
+ for (int i = 0; i < operations; i++) {
+ l.set(i);
+ t.set(String.format("%s:%05d", id, i));
+ context.write(l, t);
+ }
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/49] hadoop git commit: YARN-6471. Support to add min/max resource
configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-6471. Support to add min/max resource configuration for a queue. (Sunil G via wangda)
Change-Id: I9213f5297a6841fab5c573e85ee4c4e5f4a0b7ff
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bf4a43e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bf4a43e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bf4a43e
Branch: refs/heads/YARN-5881
Commit: 6bf4a43e74be03625fddb2da22cfd06eeaabf769
Parents: 834e91e
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 11 10:30:23 2017 -0700
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:54 2017 +0530
----------------------------------------------------------------------
.../org/apache/hadoop/util/StringUtils.java | 31 ++
.../resource/DefaultResourceCalculator.java | 6 +
.../resource/DominantResourceCalculator.java | 7 +
.../yarn/util/resource/ResourceCalculator.java | 12 +
.../hadoop/yarn/util/resource/Resources.java | 5 +
.../capacity/FifoCandidatesSelector.java | 9 +-
.../ProportionalCapacityPreemptionPolicy.java | 10 +-
.../monitor/capacity/TempQueuePerPartition.java | 16 +-
.../scheduler/AbstractResourceUsage.java | 198 +++++++
.../scheduler/QueueResourceQuotas.java | 153 ++++++
.../scheduler/ResourceUsage.java | 237 ++-------
.../scheduler/capacity/AbstractCSQueue.java | 162 +++++-
.../scheduler/capacity/CSQueue.java | 39 ++
.../scheduler/capacity/CSQueueUtils.java | 24 +-
.../CapacitySchedulerConfiguration.java | 179 ++++++-
.../scheduler/capacity/LeafQueue.java | 31 +-
.../scheduler/capacity/ParentQueue.java | 201 +++++++-
.../scheduler/capacity/UsersManager.java | 5 +-
.../PriorityUtilizationQueueOrderingPolicy.java | 11 +
.../webapp/dao/CapacitySchedulerQueueInfo.java | 15 +
.../yarn/server/resourcemanager/MockNM.java | 8 +
.../yarn/server/resourcemanager/MockRM.java | 6 +
...alCapacityPreemptionPolicyMockFramework.java | 13 +
...estProportionalCapacityPreemptionPolicy.java | 29 +-
...pacityPreemptionPolicyIntraQueueWithDRF.java | 6 +-
.../TestAbsoluteResourceConfiguration.java | 516 +++++++++++++++++++
.../capacity/TestApplicationLimits.java | 30 +-
.../TestApplicationLimitsByPartition.java | 4 +
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/capacity/TestChildQueueOrder.java | 2 +
.../scheduler/capacity/TestLeafQueue.java | 261 ++++------
.../scheduler/capacity/TestParentQueue.java | 8 +
.../scheduler/capacity/TestReservations.java | 19 +
...tPriorityUtilizationQueueOrderingPolicy.java | 3 +
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
35 files changed, 1831 insertions(+), 431 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index cda5ec7..1be8a08 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1152,4 +1152,35 @@ public class StringUtils {
return s1.equalsIgnoreCase(s2);
}
+ /**
+ * <p>Checks if the String contains only unicode letters.</p>
+ *
+ * <p><code>null</code> will return <code>false</code>.
+ * An empty String (length()=0) will return <code>true</code>.</p>
+ *
+ * <pre>
+ * StringUtils.isAlpha(null) = false
+ * StringUtils.isAlpha("") = true
+ * StringUtils.isAlpha(" ") = false
+ * StringUtils.isAlpha("abc") = true
+ * StringUtils.isAlpha("ab2c") = false
+ * StringUtils.isAlpha("ab-c") = false
+ * </pre>
+ *
+ * @param str the String to check, may be null
+ * @return <code>true</code> if only contains letters, and is non-null
+ */
+ public static boolean isAlpha(String str) {
+ if (str == null) {
+ return false;
+ }
+ int sz = str.length();
+ for (int i = 0; i < sz; i++) {
+ if (Character.isLetter(str.charAt(i)) == false) {
+ return false;
+ }
+ }
+ return true;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index 7f155e7..aefa85c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -131,4 +131,10 @@ public class DefaultResourceCalculator extends ResourceCalculator {
public boolean isAnyMajorResourceZero(Resource resource) {
return resource.getMemorySize() == 0f;
}
+
+ @Override
+ public Resource normalizeDown(Resource r, Resource stepFactor) {
+ return Resources.createResource(
+ roundDown((r.getMemorySize()), stepFactor.getMemorySize()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6b284e3..16e4527 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -567,4 +567,11 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
return false;
}
+
+ @Override
+ public Resource normalizeDown(Resource r, Resource stepFactor) {
+ return Resources.createResource(
+ roundDown(r.getMemorySize(), stepFactor.getMemorySize()),
+ roundDown(r.getVirtualCores(), stepFactor.getVirtualCores()));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index d59560f..a816290 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -235,4 +235,16 @@ public abstract class ResourceCalculator {
* @return returns true if any resource is zero.
*/
public abstract boolean isAnyMajorResourceZero(Resource resource);
+
+ /**
+ * Get resource <code>r</code>and normalize down using step-factor
+ * <code>stepFactor</code>.
+ *
+ * @param r
+ * resource to be multiplied
+ * @param stepFactor
+ * factor by which to normalize down
+ * @return resulting normalized resource
+ */
+ public abstract Resource normalizeDown(Resource r, Resource stepFactor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 068e7f1..0426ab7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -546,4 +546,9 @@ public class Resources {
Resource resource) {
return rc.isAnyMajorResourceZero(resource);
}
+
+ public static Resource normalizeDown(ResourceCalculator calculator,
+ Resource resource, Resource factor) {
+ return calculator.normalizeDown(resource, factor);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
index f843db4..748548a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java
@@ -140,10 +140,10 @@ public class FifoCandidatesSelector
// Can try preempting AMContainers (still saving atmost
// maxAMCapacityForThisQueue AMResource's) if more resources are
// required to be preemptionCandidates from this Queue.
- Resource maxAMCapacityForThisQueue = Resources.multiply(
- Resources.multiply(clusterResource,
- leafQueue.getAbsoluteCapacity()),
- leafQueue.getMaxAMResourcePerQueuePercent());
+ Resource maxAMCapacityForThisQueue = Resources
+ .multiply(
+ leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL),
+ leafQueue.getMaxAMResourcePerQueuePercent());
preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist,
resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue,
@@ -199,7 +199,6 @@ public class FifoCandidatesSelector
* Given a target preemption for a specific application, select containers
* to preempt (after unreserving all reservation for that app).
*/
- @SuppressWarnings("unchecked")
private void preemptFrom(FiCaSchedulerApp app,
Resource clusterResource, Map<String, Resource> resToObtainByPartition,
List<RMContainer> skippedAMContainerlist, Resource skippedAMSize,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 2c072d2..8327cb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingEditPolic
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
@@ -525,6 +526,13 @@ public class ProportionalCapacityPreemptionPolicy
float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt);
boolean preemptionDisabled = curQueue.getPreemptionDisabled();
+ QueueResourceQuotas queueResourceQuotas = curQueue
+ .getQueueResourceQuotas();
+ Resource effMinRes = queueResourceQuotas
+ .getEffectiveMinResource(partitionToLookAt);
+ Resource effMaxRes = queueResourceQuotas
+ .getEffectiveMaxResource(partitionToLookAt);
+
Resource current = Resources
.clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt));
Resource killable = Resources.none();
@@ -550,7 +558,7 @@ public class ProportionalCapacityPreemptionPolicy
ret = new TempQueuePerPartition(queueName, current, preemptionDisabled,
partitionToLookAt, killable, absCap, absMaxCap, partitionResource,
- reserved, curQueue);
+ reserved, curQueue, effMinRes, effMaxRes);
if (curQueue instanceof ParentQueue) {
String configuredOrderingPolicy =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index 89452f9..bd236fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -48,6 +48,9 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
double normalizedGuarantee;
+ private Resource effMinRes;
+ private Resource effMaxRes;
+
final ArrayList<TempQueuePerPartition> children;
private Collection<TempAppPerPartition> apps;
LeafQueue leafQueue;
@@ -68,7 +71,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
TempQueuePerPartition(String queueName, Resource current,
boolean preemptionDisabled, String partition, Resource killable,
float absCapacity, float absMaxCapacity, Resource totalPartitionResource,
- Resource reserved, CSQueue queue) {
+ Resource reserved, CSQueue queue, Resource effMinRes,
+ Resource effMaxRes) {
super(queueName, current, Resource.newInstance(0, 0), reserved,
Resource.newInstance(0, 0));
@@ -95,6 +99,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
this.absCapacity = absCapacity;
this.absMaxCapacity = absMaxCapacity;
this.totalPartitionResource = totalPartitionResource;
+ this.effMinRes = effMinRes;
+ this.effMaxRes = effMaxRes;
}
public void setLeafQueue(LeafQueue l) {
@@ -177,10 +183,18 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
}
public Resource getGuaranteed() {
+ if(!effMinRes.equals(Resources.none())) {
+ return Resources.clone(effMinRes);
+ }
+
return Resources.multiply(totalPartitionResource, absCapacity);
}
public Resource getMax() {
+ if(!effMaxRes.equals(Resources.none())) {
+ return Resources.clone(effMaxRes);
+ }
+
return Resources.multiply(totalPartitionResource, absMaxCapacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
new file mode 100644
index 0000000..c295323
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -0,0 +1,198 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class AbstractResourceUsage {
+ protected ReadLock readLock;
+ protected WriteLock writeLock;
+ protected Map<String, UsageByLabel> usages;
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public AbstractResourceUsage() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ usages = new HashMap<String, UsageByLabel>();
+ usages.put(NL, new UsageByLabel(NL));
+ }
+
+ // Usage enum here to make implement cleaner
+ public enum ResourceType {
+ // CACHED_USED and CACHED_PENDING may be read by anyone, but must only
+ // be written by ordering policies
+ USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING(
+ 5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE(
+ 9), EFF_MAX_RESOURCE(
+ 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12);
+
+ private int idx;
+
+ private ResourceType(int value) {
+ this.idx = value;
+ }
+ }
+
+ public static class UsageByLabel {
+ // usage by label, contains all UsageType
+ private Resource[] resArr;
+
+ public UsageByLabel(String label) {
+ resArr = new Resource[ResourceType.values().length];
+ for (int i = 0; i < resArr.length; i++) {
+ resArr[i] = Resource.newInstance(0, 0);
+ };
+ }
+
+ public Resource getUsed() {
+ return resArr[ResourceType.USED.idx];
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{used=" + resArr[0] + "%, ");
+ sb.append("pending=" + resArr[1] + "%, ");
+ sb.append("am_used=" + resArr[2] + "%, ");
+ sb.append("reserved=" + resArr[3] + "%}");
+ sb.append("min_eff=" + resArr[9] + "%, ");
+ sb.append("max_eff=" + resArr[10] + "%}");
+ sb.append("min_effup=" + resArr[11] + "%, ");
+ return sb.toString();
+ }
+ }
+
+ private static Resource normalize(Resource res) {
+ if (res == null) {
+ return Resources.none();
+ }
+ return res;
+ }
+
+ protected Resource _get(String label, ResourceType type) {
+ if (label == null) {
+ label = RMNodeLabelsManager.NO_LABEL;
+ }
+
+ try {
+ readLock.lock();
+ UsageByLabel usage = usages.get(label);
+ if (null == usage) {
+ return Resources.none();
+ }
+ return normalize(usage.resArr[type.idx]);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ protected Resource _getAll(ResourceType type) {
+ try {
+ readLock.lock();
+ Resource allOfType = Resources.createResource(0);
+ for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
+ //all usages types are initialized
+ Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
+ }
+ return allOfType;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private UsageByLabel getAndAddIfMissing(String label) {
+ if (label == null) {
+ label = RMNodeLabelsManager.NO_LABEL;
+ }
+ if (!usages.containsKey(label)) {
+ UsageByLabel u = new UsageByLabel(label);
+ usages.put(label, u);
+ return u;
+ }
+
+ return usages.get(label);
+ }
+
+ protected void _set(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.resArr[type.idx] = res;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected void _inc(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ Resources.addTo(usage.resArr[type.idx], res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected void _dec(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ Resources.subtractFrom(usage.resArr[type.idx], res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ try {
+ readLock.lock();
+ return usages.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public Set<String> getNodePartitionsSet() {
+ try {
+ readLock.lock();
+ return usages.keySet();
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
new file mode 100644
index 0000000..2e653fc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java
@@ -0,0 +1,153 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+/**
+ * QueueResourceQuotas by Labels for following fields by label
+ * - EFFECTIVE_MIN_CAPACITY
+ * - EFFECTIVE_MAX_CAPACITY
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class QueueResourceQuotas extends AbstractResourceUsage {
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public QueueResourceQuotas() {
+ super();
+ }
+
+ /*
+ * Configured Minimum Resource
+ */
+ public Resource getConfiguredMinResource() {
+ return _get(NL, ResourceType.MIN_RESOURCE);
+ }
+
+ public Resource getConfiguredMinResource(String label) {
+ return _get(label, ResourceType.MIN_RESOURCE);
+ }
+
+ public void setConfiguredMinResource(String label, Resource res) {
+ _set(label, ResourceType.MIN_RESOURCE, res);
+ }
+
+ public void setConfiguredMinResource(Resource res) {
+ _set(NL, ResourceType.MIN_RESOURCE, res);
+ }
+
+ /*
+ * Configured Maximum Resource
+ */
+ public Resource getConfiguredMaxResource() {
+ return getConfiguredMaxResource(NL);
+ }
+
+ public Resource getConfiguredMaxResource(String label) {
+ return _get(label, ResourceType.MAX_RESOURCE);
+ }
+
+ public void setConfiguredMaxResource(Resource res) {
+ setConfiguredMaxResource(NL, res);
+ }
+
+ public void setConfiguredMaxResource(String label, Resource res) {
+ _set(label, ResourceType.MAX_RESOURCE, res);
+ }
+
+ /*
+ * Effective Minimum Resource
+ */
+ public Resource getEffectiveMinResource() {
+ return _get(NL, ResourceType.EFF_MIN_RESOURCE);
+ }
+
+ public Resource getEffectiveMinResource(String label) {
+ return _get(label, ResourceType.EFF_MIN_RESOURCE);
+ }
+
+ public void setEffectiveMinResource(String label, Resource res) {
+ _set(label, ResourceType.EFF_MIN_RESOURCE, res);
+ }
+
+ public void setEffectiveMinResource(Resource res) {
+ _set(NL, ResourceType.EFF_MIN_RESOURCE, res);
+ }
+
+ /*
+ * Effective Maximum Resource
+ */
+ public Resource getEffectiveMaxResource() {
+ return getEffectiveMaxResource(NL);
+ }
+
+ public Resource getEffectiveMaxResource(String label) {
+ return _get(label, ResourceType.EFF_MAX_RESOURCE);
+ }
+
+ public void setEffectiveMaxResource(Resource res) {
+ setEffectiveMaxResource(NL, res);
+ }
+
+ public void setEffectiveMaxResource(String label, Resource res) {
+ _set(label, ResourceType.EFF_MAX_RESOURCE, res);
+ }
+
+ /*
+ * Effective Minimum Resource
+ */
+ public Resource getEffectiveMinResourceUp() {
+ return _get(NL, ResourceType.EFF_MIN_RESOURCE_UP);
+ }
+
+ public Resource getEffectiveMinResourceUp(String label) {
+ return _get(label, ResourceType.EFF_MIN_RESOURCE_UP);
+ }
+
+ public void setEffectiveMinResourceUp(String label, Resource res) {
+ _set(label, ResourceType.EFF_MIN_RESOURCE_UP, res);
+ }
+
+ public void setEffectiveMinResourceUp(Resource res) {
+ _set(NL, ResourceType.EFF_MIN_RESOURCE_UP, res);
+ }
+
+ /*
+ * Effective Maximum Resource
+ */
+ public Resource getEffectiveMaxResourceUp() {
+ return getEffectiveMaxResourceUp(NL);
+ }
+
+ public Resource getEffectiveMaxResourceUp(String label) {
+ return _get(label, ResourceType.EFF_MAX_RESOURCE_UP);
+ }
+
+ public void setEffectiveMaxResourceUp(Resource res) {
+ setEffectiveMaxResourceUp(NL, res);
+ }
+
+ public void setEffectiveMaxResourceUp(String label, Resource res) {
+ _set(label, ResourceType.EFF_MAX_RESOURCE_UP, res);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 6f0c7d2..ede4aec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -39,63 +39,12 @@ import org.apache.hadoop.yarn.util.resource.Resources;
*
* And it is thread-safe
*/
-public class ResourceUsage {
- private ReadLock readLock;
- private WriteLock writeLock;
- private Map<String, UsageByLabel> usages;
+public class ResourceUsage extends AbstractResourceUsage {
// short for no-label :)
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
- private final UsageByLabel usageNoLabel;
public ResourceUsage() {
- ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- readLock = lock.readLock();
- writeLock = lock.writeLock();
-
- usages = new HashMap<String, UsageByLabel>();
- usageNoLabel = new UsageByLabel(NL);
- usages.put(NL, usageNoLabel);
- }
-
- // Usage enum here to make implement cleaner
- private enum ResourceType {
- //CACHED_USED and CACHED_PENDING may be read by anyone, but must only
- //be written by ordering policies
- USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
- CACHED_PENDING(5), AMLIMIT(6);
-
- private int idx;
-
- private ResourceType(int value) {
- this.idx = value;
- }
- }
-
- private static class UsageByLabel {
- // usage by label, contains all UsageType
- private Resource[] resArr;
-
- public UsageByLabel(String label) {
- resArr = new Resource[ResourceType.values().length];
- for (int i = 0; i < resArr.length; i++) {
- resArr[i] = Resource.newInstance(0, 0);
- };
- }
-
- public Resource getUsed() {
- return resArr[ResourceType.USED.idx];
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("{used=" + resArr[0] + "%, ");
- sb.append("pending=" + resArr[1] + "%, ");
- sb.append("am_used=" + resArr[2] + "%, ");
- sb.append("reserved=" + resArr[3] + "%}");
- sb.append("am_limit=" + resArr[6] + "%, ");
- return sb.toString();
- }
+ super();
}
/*
@@ -109,22 +58,6 @@ public class ResourceUsage {
return _get(label, ResourceType.USED);
}
- public Resource getCachedUsed() {
- return _get(NL, ResourceType.CACHED_USED);
- }
-
- public Resource getCachedUsed(String label) {
- return _get(label, ResourceType.CACHED_USED);
- }
-
- public Resource getCachedPending() {
- return _get(NL, ResourceType.CACHED_PENDING);
- }
-
- public Resource getCachedPending(String label) {
- return _get(label, ResourceType.CACHED_PENDING);
- }
-
public void incUsed(String label, Resource res) {
_inc(label, ResourceType.USED, res);
}
@@ -145,7 +78,7 @@ public class ResourceUsage {
setUsed(NL, res);
}
- public void copyAllUsed(ResourceUsage other) {
+ public void copyAllUsed(AbstractResourceUsage other) {
try {
writeLock.lock();
for (Entry<String, UsageByLabel> entry : other.usages.entrySet()) {
@@ -160,22 +93,6 @@ public class ResourceUsage {
_set(label, ResourceType.USED, res);
}
- public void setCachedUsed(String label, Resource res) {
- _set(label, ResourceType.CACHED_USED, res);
- }
-
- public void setCachedUsed(Resource res) {
- _set(NL, ResourceType.CACHED_USED, res);
- }
-
- public void setCachedPending(String label, Resource res) {
- _set(label, ResourceType.CACHED_PENDING, res);
- }
-
- public void setCachedPending(Resource res) {
- _set(NL, ResourceType.CACHED_PENDING, res);
- }
-
/*
* Pending
*/
@@ -281,6 +198,47 @@ public class ResourceUsage {
_set(label, ResourceType.AMUSED, res);
}
+ public Resource getAllPending() {
+ return _getAll(ResourceType.PENDING);
+ }
+
+ public Resource getAllUsed() {
+ return _getAll(ResourceType.USED);
+ }
+
+ // Cache Used
+ public Resource getCachedUsed() {
+ return _get(NL, ResourceType.CACHED_USED);
+ }
+
+ public Resource getCachedUsed(String label) {
+ return _get(label, ResourceType.CACHED_USED);
+ }
+
+ public Resource getCachedPending() {
+ return _get(NL, ResourceType.CACHED_PENDING);
+ }
+
+ public Resource getCachedPending(String label) {
+ return _get(label, ResourceType.CACHED_PENDING);
+ }
+
+ public void setCachedUsed(String label, Resource res) {
+ _set(label, ResourceType.CACHED_USED, res);
+ }
+
+ public void setCachedUsed(Resource res) {
+ _set(NL, ResourceType.CACHED_USED, res);
+ }
+
+ public void setCachedPending(String label, Resource res) {
+ _set(label, ResourceType.CACHED_PENDING, res);
+ }
+
+ public void setCachedPending(Resource res) {
+ _set(NL, ResourceType.CACHED_PENDING, res);
+ }
+
/*
* AM-Resource Limit
*/
@@ -316,94 +274,6 @@ public class ResourceUsage {
_set(label, ResourceType.AMLIMIT, res);
}
- private static Resource normalize(Resource res) {
- if (res == null) {
- return Resources.none();
- }
- return res;
- }
-
- private Resource _get(String label, ResourceType type) {
- if (label == null || label.equals(NL)) {
- return normalize(usageNoLabel.resArr[type.idx]);
- }
- try {
- readLock.lock();
- UsageByLabel usage = usages.get(label);
- if (null == usage) {
- return Resources.none();
- }
- return normalize(usage.resArr[type.idx]);
- } finally {
- readLock.unlock();
- }
- }
-
- private Resource _getAll(ResourceType type) {
- try {
- readLock.lock();
- Resource allOfType = Resources.createResource(0);
- for (Map.Entry<String, UsageByLabel> usageEntry : usages.entrySet()) {
- //all usages types are initialized
- Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]);
- }
- return allOfType;
- } finally {
- readLock.unlock();
- }
- }
-
- public Resource getAllPending() {
- return _getAll(ResourceType.PENDING);
- }
-
- public Resource getAllUsed() {
- return _getAll(ResourceType.USED);
- }
-
- private UsageByLabel getAndAddIfMissing(String label) {
- if (label == null || label.equals(NL)) {
- return usageNoLabel;
- }
- if (!usages.containsKey(label)) {
- UsageByLabel u = new UsageByLabel(label);
- usages.put(label, u);
- return u;
- }
-
- return usages.get(label);
- }
-
- private void _set(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- usage.resArr[type.idx] = res;
- } finally {
- writeLock.unlock();
- }
- }
-
- private void _inc(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- Resources.addTo(usage.resArr[type.idx], res);
- } finally {
- writeLock.unlock();
- }
- }
-
- private void _dec(String label, ResourceType type, Resource res) {
- try {
- writeLock.lock();
- UsageByLabel usage = getAndAddIfMissing(label);
- Resources.subtractFrom(usage.resArr[type.idx], res);
- } finally {
- writeLock.unlock();
- }
- }
-
public Resource getCachedDemand(String label) {
try {
readLock.lock();
@@ -415,23 +285,4 @@ public class ResourceUsage {
readLock.unlock();
}
}
-
- @Override
- public String toString() {
- try {
- readLock.lock();
- return usages.toString();
- } finally {
- readLock.unlock();
- }
- }
-
- public Set<String> getNodePartitionsSet() {
- try {
- readLock.lock();
- return usages.keySet();
- } finally {
- readLock.unlock();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 74c85ce..dddac4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@@ -87,6 +89,7 @@ public abstract class AbstractCSQueue implements CSQueue {
final ResourceCalculator resourceCalculator;
Set<String> accessibleLabels;
+ Set<String> resourceTypes;
final RMNodeLabelsManager labelManager;
String defaultLabelExpression;
@@ -102,6 +105,14 @@ public abstract class AbstractCSQueue implements CSQueue {
// etc.
QueueCapacities queueCapacities;
+ QueueResourceQuotas queueResourceQuotas;
+
+ protected enum CapacityConfigType {
+ NONE, PERCENTAGE, ABSOLUTE_RESOURCE
+ };
+ protected CapacityConfigType capacityConfigType =
+ CapacityConfigType.NONE;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
protected CapacitySchedulerContext csContext;
@@ -141,6 +152,9 @@ public abstract class AbstractCSQueue implements CSQueue {
// initialize QueueCapacities
queueCapacities = new QueueCapacities(parent == null);
+ // initialize queueResourceQuotas
+ queueResourceQuotas = new QueueResourceQuotas();
+
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
@@ -276,6 +290,10 @@ public abstract class AbstractCSQueue implements CSQueue {
this.defaultLabelExpression =
csContext.getConfiguration().getDefaultNodeLabelExpression(
getQueuePath());
+ this.resourceTypes = new HashSet<String>();
+ for (AbsoluteResourceType type : AbsoluteResourceType.values()) {
+ resourceTypes.add(type.toString().toLowerCase());
+ }
// inherit from parent if labels not set
if (this.accessibleLabels == null && parent != null) {
@@ -292,6 +310,11 @@ public abstract class AbstractCSQueue implements CSQueue {
// After we setup labels, we can setup capacities
setupConfigurableCapacities();
+ // Also fetch minimum/maximum resource constraint for this queue if
+ // configured.
+ capacityConfigType = CapacityConfigType.NONE;
+ updateConfigurableResourceRequirement(getQueuePath(), clusterResource);
+
this.maximumAllocation =
csContext.getConfiguration().getMaximumAllocationPerQueue(
getQueuePath());
@@ -364,6 +387,125 @@ public abstract class AbstractCSQueue implements CSQueue {
return unionInheritedWeights;
}
+ protected void updateConfigurableResourceRequirement(String queuePath,
+ Resource clusterResource) {
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ Set<String> configuredNodelabels = conf.getConfiguredNodeLabels(queuePath);
+
+ for (String label : configuredNodelabels) {
+ Resource minResource = conf.getMinimumResourceRequirement(label,
+ queuePath, resourceTypes);
+ Resource maxResource = conf.getMaximumResourceRequirement(label,
+ queuePath, resourceTypes);
+
+ if (this.capacityConfigType.equals(CapacityConfigType.NONE)) {
+ this.capacityConfigType = (!minResource.equals(Resources.none())
+ && queueCapacities.getAbsoluteCapacity(label) == 0f)
+ ? CapacityConfigType.ABSOLUTE_RESOURCE
+ : CapacityConfigType.PERCENTAGE;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("capacityConfigType is updated as '" + capacityConfigType
+ + "' for queue '" + getQueueName());
+ }
+ }
+
+ validateAbsoluteVsPercentageCapacityConfig(minResource);
+
+ // If min resource for a resource type is greater than its max resource,
+ // throw exception to handle such error configs.
+ if (!maxResource.equals(Resources.none()) && Resources.greaterThan(
+ resourceCalculator, clusterResource, minResource, maxResource)) {
+ throw new IllegalArgumentException("Min resource configuration "
+ + minResource + " is greater than its max value:" + maxResource
+ + " in queue:" + getQueueName());
+ }
+
+ // If parent's max resource is lesser to a specific child's max
+ // resource, throw exception to handle such error configs.
+ if (parent != null) {
+ Resource parentMaxRes = parent.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ parentMaxRes, Resources.none())) {
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ maxResource, parentMaxRes)) {
+ throw new IllegalArgumentException("Max resource configuration "
+ + maxResource + " is greater than parents max value:"
+ + parentMaxRes + " in queue:" + getQueueName());
+ }
+
+ // If child's max resource is not set, but its parent max resource is
+ // set, we must set child max resource to its parent's.
+ if (maxResource.equals(Resources.none())
+ && !minResource.equals(Resources.none())) {
+ maxResource = Resources.clone(parentMaxRes);
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating absolute resource configuration for queue:"
+ + getQueueName() + " as minResource=" + minResource
+ + " and maxResource=" + maxResource);
+ }
+
+ queueResourceQuotas.setConfiguredMinResource(label, minResource);
+ queueResourceQuotas.setConfiguredMaxResource(label, maxResource);
+ }
+ }
+
+ private void validateAbsoluteVsPercentageCapacityConfig(
+ Resource minResource) {
+ CapacityConfigType localType = CapacityConfigType.PERCENTAGE;
+ if (!minResource.equals(Resources.none())) {
+ localType = CapacityConfigType.ABSOLUTE_RESOURCE;
+ }
+
+ if (!queueName.equals("root")
+ && !this.capacityConfigType.equals(localType)) {
+ throw new IllegalArgumentException("Queue '" + getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource.");
+ }
+ }
+
+ @Override
+ public CapacityConfigType getCapacityConfigType() {
+ return capacityConfigType;
+ }
+
+ @Override
+ public Resource getEffectiveCapacity(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMinResource(label));
+ }
+
+ @Override
+ public Resource getEffectiveCapacityUp(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label));
+ }
+
+ @Override
+ public Resource getEffectiveCapacityDown(String label, Resource factor) {
+ return Resources.normalizeDown(resourceCalculator,
+ getQueueResourceQuotas().getEffectiveMinResource(label),
+ minimumAllocation);
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacity(String label) {
+ return Resources
+ .clone(getQueueResourceQuotas().getEffectiveMaxResource(label));
+ }
+
+ @Override
+ public Resource getEffectiveMaxCapacityDown(String label, Resource factor) {
+ return Resources.normalizeDown(resourceCalculator,
+ getQueueResourceQuotas().getEffectiveMaxResource(label),
+ minimumAllocation);
+ }
+
private void initializeQueueState(QueueState previousState,
QueueState configuredState, QueueState parentState) {
// verify that we can not any value for State other than RUNNING/STOPPED
@@ -555,6 +697,11 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
+ public QueueResourceQuotas getQueueResourceQuotas() {
+ return queueResourceQuotas;
+ }
+
+ @Override
public ReentrantReadWriteLock.ReadLock getReadLock() {
return readLock;
}
@@ -604,7 +751,7 @@ public abstract class AbstractCSQueue implements CSQueue {
* limit-set-by-parent)
*/
Resource queueMaxResource =
- getQueueMaxResource(nodePartition, clusterResource);
+ getQueueMaxResource(nodePartition);
return Resources.min(resourceCalculator, clusterResource,
queueMaxResource, currentResourceLimits.getLimit());
@@ -617,11 +764,8 @@ public abstract class AbstractCSQueue implements CSQueue {
return Resources.none();
}
- Resource getQueueMaxResource(String nodePartition, Resource clusterResource) {
- return Resources.multiplyAndNormalizeDown(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- queueCapacities.getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource getQueueMaxResource(String nodePartition) {
+ return getEffectiveMaxCapacity(nodePartition);
}
public boolean hasChildQueues() {
@@ -782,7 +926,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.incUsed(nodeLabel, resourceToInc);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- nodeLabel, this);
+ Resources.none(), nodeLabel, this);
if (null != parent) {
parent.incUsedResource(nodeLabel, resourceToInc, null);
}
@@ -798,7 +942,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueUsage.decUsed(nodeLabel, resourceToDec);
CSQueueUtils.updateUsedCapacity(resourceCalculator,
labelManager.getResourceByLabel(nodeLabel, Resources.none()),
- nodeLabel, this);
+ Resources.none(), nodeLabel, this);
if (null != parent) {
parent.decUsedResource(nodeLabel, resourceToDec, null);
}
@@ -904,7 +1048,7 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maxResourceLimit;
if (allocation.getSchedulingMode()
== SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) {
- maxResourceLimit = getQueueMaxResource(partition, cluster);
+ maxResourceLimit = getQueueMaxResource(partition);
} else{
maxResourceLimit = labelManager.getResourceByLabel(
schedulerContainer.getNodePartition(), cluster);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 43e7f53..2e29a71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -41,10 +41,12 @@ import org.apache.hadoop.yarn.security.PrivilegedEntity;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -355,4 +357,41 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
* @return map of usernames and corresponding weight
*/
Map<String, Float> getUserWeights();
+
+ /**
+ * Get QueueResourceQuotas associated with each queue.
+ * @return QueueResourceQuotas
+ */
+ public QueueResourceQuotas getQueueResourceQuotas();
+
+ /**
+ * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE
+ * @return CapacityConfigType
+ */
+ public CapacityConfigType getCapacityConfigType();
+
+ /**
+ * Get effective capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @return effective queue capacity
+ */
+ Resource getEffectiveCapacity(String label);
+ Resource getEffectiveCapacityUp(String label);
+ Resource getEffectiveCapacityDown(String label, Resource factor);
+
+ /**
+ * Get effective max capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @return effective max queue capacity
+ */
+ Resource getEffectiveMaxCapacity(String label);
+ Resource getEffectiveMaxCapacityDown(String label, Resource factor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index e1014c1..81dec80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -150,7 +150,7 @@ class CSQueueUtils {
}
}
}
-
+
// Set absolute capacities for {capacity, maximum-capacity}
private static void updateAbsoluteCapacitiesByNodeLabels(
QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
@@ -180,8 +180,8 @@ class CSQueueUtils {
* used resource for all partitions of this queue.
*/
public static void updateUsedCapacity(final ResourceCalculator rc,
- final Resource totalPartitionResource, String nodePartition,
- AbstractCSQueue childQueue) {
+ final Resource totalPartitionResource, Resource clusterResource,
+ String nodePartition, AbstractCSQueue childQueue) {
QueueCapacities queueCapacities = childQueue.getQueueCapacities();
CSQueueMetrics queueMetrics = childQueue.getMetrics();
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
@@ -193,11 +193,8 @@ class CSQueueUtils {
if (Resources.greaterThan(rc, totalPartitionResource,
totalPartitionResource, Resources.none())) {
- // queueGuaranteed = totalPartitionedResource *
- // absolute_capacity(partition)
- Resource queueGuranteedResource =
- Resources.multiply(totalPartitionResource,
- queueCapacities.getAbsoluteCapacity(nodePartition));
+ Resource queueGuranteedResource = childQueue
+ .getEffectiveCapacity(nodePartition);
// make queueGuranteed >= minimum_allocation to avoid divided by 0.
queueGuranteedResource =
@@ -248,9 +245,7 @@ class CSQueueUtils {
for (String partition : nodeLabels) {
// Calculate guaranteed resource for a label in a queue by below logic.
// (total label resource) * (absolute capacity of label in that queue)
- Resource queueGuranteedResource = Resources.multiply(nlm
- .getResourceByLabel(partition, cluster), queue.getQueueCapacities()
- .getAbsoluteCapacity(partition));
+ Resource queueGuranteedResource = queue.getEffectiveCapacity(partition);
// Available resource in queue for a specific label will be calculated as
// {(guaranteed resource for a label in a queue) -
@@ -289,15 +284,14 @@ class CSQueueUtils {
ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage();
if (nodePartition == null) {
- for (String partition : Sets.union(
- queueCapacities.getNodePartitionsSet(),
+ for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
queueResourceUsage.getNodePartitionsSet())) {
updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster),
- partition, childQueue);
+ cluster, partition, childQueue);
}
} else {
updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster),
- nodePartition, childQueue);
+ cluster, nodePartition, childQueue);
}
// Update queue metrics w.r.t node labels. In a generic way, we can
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 4515453..f81e097 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FairOrderi
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -60,6 +61,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.Set;
import java.util.StringTokenizer;
@@ -317,6 +320,21 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
+ /** Configuring absolute min/max resources in a queue **/
+ @Private
+ public static final String MINIMUM_RESOURCE = "min-resource";
+
+ @Private
+ public static final String MAXIMUM_RESOURCE = "max-resource";
+
+ public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores";
+
+ public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)";
+
+ public enum AbsoluteResourceType {
+ MEMORY, VCORES;
+ }
+
AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser();
public CapacitySchedulerConfiguration() {
@@ -394,7 +412,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public float getNonLabeledQueueCapacity(String queue) {
float capacity = queue.equals("root") ? 100.0f : getFloat(
- getQueuePrefix(queue) + CAPACITY, UNDEFINED);
+ getQueuePrefix(queue) + CAPACITY, 0f);
if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
throw new IllegalArgumentException("Illegal " +
"capacity of " + capacity + " for queue " + queue);
@@ -1676,4 +1694,163 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
queuePath);
setMaximumCapacity(leafQueueConfPrefix, val);
}
+
+ public static String getUnits(String resourceValue) {
+ String units;
+ for (int i = 0; i < resourceValue.length(); i++) {
+ if (Character.isAlphabetic(resourceValue.charAt(i))) {
+ units = resourceValue.substring(i);
+ if (StringUtils.isAlpha(units)) {
+ return units;
+ }
+ }
+ }
+ return "";
+ }
+
+ /**
+ * Get absolute minimum resource requirement for a queue.
+ *
+ * @param label
+ * NodeLabel
+ * @param queue
+ * queue path
+ * @param resourceTypes
+ * Resource types
+ * @return ResourceInformation
+ */
+ public Resource getMinimumResourceRequirement(String label, String queue,
+ Set<String> resourceTypes) {
+ return internalGetLabeledResourceRequirementForQueue(queue, label,
+ resourceTypes, MINIMUM_RESOURCE);
+ }
+
+ /**
+ * Get absolute maximum resource requirement for a queue.
+ *
+ * @param label
+ * NodeLabel
+ * @param queue
+ * queue path
+ * @param resourceTypes
+ * Resource types
+ * @return Resource
+ */
+ public Resource getMaximumResourceRequirement(String label, String queue,
+ Set<String> resourceTypes) {
+ return internalGetLabeledResourceRequirementForQueue(queue, label,
+ resourceTypes, MAXIMUM_RESOURCE);
+ }
+
+ @VisibleForTesting
+ public void setMinimumResourceRequirement(String label, String queue,
+ Resource resource) {
+ updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE);
+ }
+
+ @VisibleForTesting
+ public void setMaximumResourceRequirement(String label, String queue,
+ Resource resource) {
+ updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE);
+ }
+
+ private void updateMinMaxResourceToConf(String label, String queue,
+ Resource resource, String type) {
+ if (queue.equals("root")) {
+ throw new IllegalArgumentException(
+ "Cannot set resource, root queue will take 100% of cluster capacity");
+ }
+
+ StringBuilder resourceString = new StringBuilder();
+ resourceString
+ .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "="
+ + resource.getMemorySize() + ","
+ + AbsoluteResourceType.VCORES.toString().toLowerCase() + "="
+ + resource.getVirtualCores() + "]");
+
+ String prefix = getQueuePrefix(queue) + type;
+ if (!label.isEmpty()) {
+ prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label
+ + DOT + type;
+ }
+ set(prefix, resourceString.toString());
+ }
+
+ private Resource internalGetLabeledResourceRequirementForQueue(String queue,
+ String label, Set<String> resourceTypes, String suffix) {
+ String propertyName = getNodeLabelPrefix(queue, label) + suffix;
+ String resourceString = get(propertyName);
+ if (resourceString == null || resourceString.isEmpty()) {
+ return Resources.none();
+ }
+
+ // Define resource here.
+ Resource resource = Resource.newInstance(0l, 0);
+ Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE)
+ .matcher(resourceString);
+ /*
+ * Absolute resource configuration for a queue will be grouped by "[]".
+ * Syntax of absolute resource config could be like below
+ * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores".
+ */
+ if (matcher.find()) {
+ // Get the sub-group.
+ String subGroup = matcher.group(1);
+ if (subGroup.trim().isEmpty()) {
+ return Resources.none();
+ }
+
+ for (String kvPair : subGroup.trim().split(",")) {
+ String[] splits = kvPair.split("=");
+
+ // Ensure that each sub string is key value pair separated by '='.
+ if (splits != null && splits.length > 1) {
+ updateResourceValuesFromConfig(resourceTypes, resource, splits);
+ }
+ }
+ }
+
+ // Memory has to be configured always.
+ if (resource.getMemorySize() == 0l) {
+ return Resources.none();
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CSConf - getAbsolueResourcePerQueue: prefix="
+ + getNodeLabelPrefix(queue, label) + ", capacity=" + resource);
+ }
+ return resource;
+ }
+
+ private void updateResourceValuesFromConfig(Set<String> resourceTypes,
+ Resource resource, String[] splits) {
+
+ // If key is not a valid type, skip it.
+ if (!resourceTypes.contains(splits[0])) {
+ return;
+ }
+
+ String units = getUnits(splits[1]);
+ Long resourceValue = Long
+ .valueOf(splits[1].substring(0, splits[1].length() - units.length()));
+
+ // Convert all incoming units to MB if units is configured.
+ if (!units.isEmpty()) {
+ resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue);
+ }
+
+ // map it based on key.
+ AbsoluteResourceType resType = AbsoluteResourceType
+ .valueOf(StringUtils.toUpperCase(splits[0].trim()));
+ switch (resType) {
+ case MEMORY :
+ resource.setMemorySize(resourceValue);
+ break;
+ case VCORES :
+ resource.setVirtualCores(resourceValue.intValue());
+ break;
+ default :
+ break;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index e8342d9..1460121 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -671,12 +671,7 @@ public class LeafQueue extends AbstractCSQueue {
1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1));
effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f);
- Resource queuePartitionResource = Resources
- .multiplyAndNormalizeUp(resourceCalculator,
- labelManager.getResourceByLabel(nodePartition,
- lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition),
- minimumAllocation);
+ Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
Resource userAMLimit = Resources.multiplyAndNormalizeUp(
resourceCalculator, queuePartitionResource,
@@ -705,11 +700,7 @@ public class LeafQueue extends AbstractCSQueue {
* non-labeled), * with per-partition am-resource-percent to get the max am
* resource limit for this queue and partition.
*/
- Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition),
- minimumAllocation);
+ Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition);
Resource queueCurrentLimit = Resources.none();
// For non-labeled partition, we need to consider the current queue
@@ -965,6 +956,14 @@ public class LeafQueue extends AbstractCSQueue {
private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) {
// Set preemption-allowed:
// For leaf queue, only under-utilized queue is allowed to preempt resources from other queues
+ if (!queueResourceQuotas.getEffectiveMinResource(nodePartition)
+ .equals(Resources.none())) {
+ limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator,
+ csContext.getClusterResource(), queueUsage.getUsed(nodePartition),
+ queueResourceQuotas.getEffectiveMinResource(nodePartition)));
+ return;
+ }
+
float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition);
float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition);
limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity);
@@ -1344,7 +1343,7 @@ public class LeafQueue extends AbstractCSQueue {
currentPartitionResourceLimit =
partition.equals(RMNodeLabelsManager.NO_LABEL)
? currentPartitionResourceLimit
- : getQueueMaxResource(partition, clusterResource);
+ : getQueueMaxResource(partition);
Resource headroom = Resources.componentwiseMin(
Resources.subtract(userLimitResource, user.getUsed(partition)),
@@ -1716,12 +1715,8 @@ public class LeafQueue extends AbstractCSQueue {
// this. So need cap limits by queue's max capacity here.
this.cachedResourceLimitsForHeadroom =
new ResourceLimits(currentResourceLimits.getLimit());
- Resource queueMaxResource =
- Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
- .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
- queueCapacities
- .getAbsoluteMaximumCapacity(RMNodeLabelsManager.NO_LABEL),
- minimumAllocation);
+ Resource queueMaxResource = getEffectiveMaxCapacityDown(
+ RMNodeLabelsManager.NO_LABEL, minimumAllocation);
this.cachedResourceLimitsForHeadroom.setLimit(Resources.min(
resourceCalculator, clusterResource, queueMaxResource,
currentResourceLimits.getLimit()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 959ca51..c770fac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -162,31 +164,78 @@ public class ParentQueue extends AbstractCSQueue {
writeLock.lock();
// Validate
float childCapacities = 0;
+ Resource minResDefaultLabel = Resources.createResource(0, 0);
for (CSQueue queue : childQueues) {
childCapacities += queue.getCapacity();
+ Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource());
+
+ // If any child queue is using percentage based capacity model vs parent
+ // queues' absolute configuration or vice versa, throw back an
+ // exception.
+ if (!queueName.equals("root") && getCapacity() != 0f
+ && !queue.getQueueResourceQuotas().getConfiguredMinResource()
+ .equals(Resources.none())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + " configuration or absolute resource together.");
+ }
}
+
float delta = Math.abs(1.0f - childCapacities); // crude way to check
// allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || (
- (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
- throw new IllegalArgumentException(
- "Illegal" + " capacity of " + childCapacities
- + " for children of queue " + queueName);
+ if ((minResDefaultLabel.equals(Resources.none())
+ && (queueCapacities.getCapacity() > 0) && (delta > PRECISION))
+ || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
+ throw new IllegalArgumentException("Illegal" + " capacity of "
+ + childCapacities + " for children of queue " + queueName);
}
// check label capacities
for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
// check children's labels
float sum = 0;
+ Resource minRes = Resources.createResource(0, 0);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel,
+ scheduler.getClusterResource());
for (CSQueue queue : childQueues) {
sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+
+ // If any child queue of a label is using percentage based capacity
+ // model vs parent queues' absolute configuration or vice versa, throw
+ // back an exception
+ if (!queueName.equals("root") && !this.capacityConfigType
+ .equals(queue.getCapacityConfigType())) {
+ throw new IllegalArgumentException("Parent queue '" + getQueueName()
+ + "' and child queue '" + queue.getQueueName()
+ + "' should use either percentage based capacity"
+ + "configuration or absolute resource together for label:"
+ + nodeLabel);
+ }
+
+ // Accumulate all min/max resource configured for all child queues.
+ Resources.addTo(minRes, queue.getQueueResourceQuotas()
+ .getConfiguredMinResource(nodeLabel));
}
- if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
+ if ((minResDefaultLabel.equals(Resources.none()) && capacityByLabel > 0
+ && Math.abs(1.0f - sum) > PRECISION)
|| (capacityByLabel == 0) && (sum > 0)) {
throw new IllegalArgumentException(
"Illegal" + " capacity of " + sum + " for children of queue "
+ queueName + " for label=" + nodeLabel);
}
+
+ // Ensure that for each parent queue: parent.min-resource >=
+ // Σ(child.min-resource).
+ Resource parentMinResource = queueResourceQuotas
+ .getConfiguredMinResource(nodeLabel);
+ if (!parentMinResource.equals(Resources.none()) && Resources.lessThan(
+ resourceCalculator, resourceByLabel, parentMinResource, minRes)) {
+ throw new IllegalArgumentException("Parent Queues" + " capacity: "
+ + parentMinResource + " is less than" + " to its children:"
+ + minRes + " for queue:" + queueName);
+ }
}
this.childQueues.clear();
@@ -687,11 +736,8 @@ public class ParentQueue extends AbstractCSQueue {
child.getQueueResourceUsage().getUsed(nodePartition));
// Get child's max resource
- Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, clusterResource),
- child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition),
- minimumAllocation);
+ Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown(
+ nodePartition, minimumAllocation);
// Child's limit should be capped by child configured max resource
childLimit =
@@ -827,6 +873,14 @@ public class ParentQueue extends AbstractCSQueue {
ResourceLimits resourceLimits) {
try {
writeLock.lock();
+
+ // Update effective capacity in all parent queue.
+ Set<String> configuredNodelabels = csContext.getConfiguration()
+ .getConfiguredNodeLabels(getQueuePath());
+ for (String label : configuredNodelabels) {
+ calculateEffectiveResourcesAndCapacity(label, clusterResource);
+ }
+
// Update all children
for (CSQueue childQueue : childQueues) {
// Get ResourceLimits of child queue before assign containers
@@ -848,6 +902,110 @@ public class ParentQueue extends AbstractCSQueue {
return true;
}
+ private void calculateEffectiveResourcesAndCapacity(String label,
+ Resource clusterResource) {
+
+ // For root queue, ensure that max/min resource is updated to latest
+ // cluster resource.
+ Resource resourceByLabel = labelManager.getResourceByLabel(label,
+ clusterResource);
+ if (getQueueName().equals("root")) {
+ queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel);
+ queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel);
+ queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel);
+ }
+
+ // Total configured min resources of direct children of queue
+ Resource configuredMinResources = Resource.newInstance(0L, 0);
+ for (CSQueue childQueue : getChildQueues()) {
+ Resources.addTo(configuredMinResources,
+ childQueue.getQueueResourceQuotas().getConfiguredMinResource(label));
+ }
+
+ // Factor to scale down effective resource: When cluster has sufficient
+ // resources, effective_min_resources will be same as configured
+ // min_resources.
+ float effectiveMinRatio = 1;
+ ResourceCalculator rc = this.csContext.getResourceCalculator();
+ if (getQueueName().equals("root")) {
+ if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc,
+ clusterResource, resourceByLabel, configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ resourceByLabel, configuredMinResources);
+ }
+ } else {
+ if (Resources.lessThan(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources)) {
+ effectiveMinRatio = Resources.divide(rc, clusterResource,
+ queueResourceQuotas.getEffectiveMinResource(label),
+ configuredMinResources);
+ }
+ }
+
+ // loop and do this for all child queues
+ for (CSQueue childQueue : getChildQueues()) {
+ Resource minResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMinResource(label);
+
+ // Update effective resource (min/max) to each child queue.
+ if (childQueue.getCapacityConfigType()
+ .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(minResource, effectiveMinRatio));
+
+ // Max resource of a queue should be a minimum of {configuredMaxRes,
+ // parentMaxRes}. parentMaxRes could be configured value. But if not
+ // present could also be taken from effective max resource of parent.
+ Resource parentMaxRes = queueResourceQuotas
+ .getConfiguredMaxResource(label);
+ if (parentMaxRes.equals(Resources.none())) {
+ parentMaxRes = parent.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label);
+ }
+
+ // Minimum of {childMaxResource, parentMaxRes}. However if
+ // childMaxResource is empty, consider parent's max resource alone.
+ Resource childMaxResource = childQueue.getQueueResourceQuotas()
+ .getConfiguredMaxResource(label);
+ Resource effMaxResource = Resources.min(resourceCalculator,
+ resourceByLabel, childMaxResource.equals(Resources.none())
+ ? parentMaxRes
+ : childMaxResource,
+ parentMaxRes);
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.clone(effMaxResource));
+ } else {
+ childQueue.getQueueResourceQuotas().setEffectiveMinResource(label,
+ Resources.multiply(resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label)));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label,
+ Resources.multiply(resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label)));
+
+ childQueue.getQueueResourceQuotas().setEffectiveMinResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc, resourceByLabel,
+ childQueue.getQueueCapacities().getAbsoluteCapacity(label),
+ minimumAllocation));
+ childQueue.getQueueResourceQuotas().setEffectiveMaxResourceUp(label,
+ Resources.multiplyAndNormalizeUp(rc,
+ resourceByLabel, childQueue.getQueueCapacities()
+ .getAbsoluteMaximumCapacity(label),
+ minimumAllocation));
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Updating effective min resource for queue:"
+ + childQueue.getQueueName() + " as effMinResource="
+ + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label)
+ + "and Updating effective max resource as effMaxResource="
+ + childQueue.getQueueResourceQuotas()
+ .getEffectiveMaxResource(label));
+ }
+ }
+ }
+
@Override
public List<CSQueue> getChildQueues() {
try {
@@ -980,9 +1138,21 @@ public class ParentQueue extends AbstractCSQueue {
* When this happens, we have to preempt killable container (on same or different
* nodes) of parent queue to avoid violating parent's max resource.
*/
- if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
- < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
- killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+ if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition)
+ .equals(Resources.none())) {
+ if (Resources.lessThan(resourceCalculator, clusterResource,
+ queueResourceQuotas.getEffectiveMaxResource(nodePartition),
+ queueUsage.getUsed(nodePartition))) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
+ } else {
+ if (getQueueCapacities()
+ .getAbsoluteMaximumCapacity(nodePartition) < getQueueCapacities()
+ .getAbsoluteUsedCapacity(nodePartition)) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition,
+ clusterResource);
+ }
}
} finally {
writeLock.unlock();
@@ -999,8 +1169,7 @@ public class ParentQueue extends AbstractCSQueue {
Resource partitionResource = labelManager.getResourceByLabel(partition,
null);
- Resource maxResource = Resources.multiply(partitionResource,
- getQueueCapacities().getAbsoluteMaximumCapacity(partition));
+ Resource maxResource = getEffectiveMaxCapacity(partition);
while (Resources.greaterThan(resourceCalculator, partitionResource,
queueUsage.getUsed(partition), maxResource)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 33f30b0..efc20e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -686,10 +686,7 @@ public class UsersManager implements AbstractUsersManager {
* * If we're running over capacity, then its (usedResources + required)
* (which extra resources we are allocating)
*/
- Resource queueCapacity = Resources.multiplyAndNormalizeUp(
- resourceCalculator, partitionResource,
- lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition),
- lQueue.getMinimumAllocation());
+ Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition);
/*
* Assume we have required resource equals to minimumAllocation, this can
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/49] hadoop git commit: HDFS-12804. Use slf4j instead of log4j in
FSEditLog. Contributed by Mukul Kumar Singh.
Posted by su...@apache.org.
HDFS-12804. Use slf4j instead of log4j in FSEditLog. Contributed by Mukul Kumar Singh.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60fc2a13
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60fc2a13
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60fc2a13
Branch: refs/heads/YARN-5881
Commit: 60fc2a138827c2c29fa7e9d6844e3b8d43809726
Parents: 0d781dd
Author: Chen Liang <cl...@apache.org>
Authored: Mon Nov 20 12:49:53 2017 -0800
Committer: Chen Liang <cl...@apache.org>
Committed: Mon Nov 20 12:49:53 2017 -0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSEditLog.java | 23 ++++++++++----------
.../hdfs/server/namenode/TestEditLog.java | 4 ++--
.../server/namenode/TestEditLogAutoroll.java | 10 ++++-----
.../hdfs/server/namenode/TestEditLogRace.java | 4 ++--
.../server/namenode/ha/TestEditLogTailer.java | 8 +++----
5 files changed, 24 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fc2a13/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 7ca63f8..72e00ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -29,8 +29,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -114,6 +112,8 @@ import org.apache.hadoop.security.token.delegation.DelegationKey;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* FSEditLog maintains a log of the namespace modifications.
@@ -122,9 +122,7 @@ import com.google.common.collect.Lists;
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class FSEditLog implements LogsPurgeable {
-
- public static final Log LOG = LogFactory.getLog(FSEditLog.class);
-
+ public static final Logger LOG = LoggerFactory.getLogger(FSEditLog.class);
/**
* State machine for edit log.
*
@@ -329,7 +327,8 @@ public class FSEditLog implements LogsPurgeable {
String error = String.format("Cannot start writing at txid %s " +
"when there is a stream available for read: %s",
segmentTxId, streams.get(0));
- IOUtils.cleanup(LOG, streams.toArray(new EditLogInputStream[0]));
+ IOUtils.cleanupWithLogger(LOG,
+ streams.toArray(new EditLogInputStream[0]));
throw new IllegalStateException(error);
}
@@ -689,9 +688,9 @@ public class FSEditLog implements LogsPurgeable {
"Could not sync enough journals to persistent storage " +
"due to " + e.getMessage() + ". " +
"Unsynced transactions: " + (txid - synctxid);
- LOG.fatal(msg, new Exception());
+ LOG.error(msg, new Exception());
synchronized(journalSetLock) {
- IOUtils.cleanup(LOG, journalSet);
+ IOUtils.cleanupWithLogger(LOG, journalSet);
}
terminate(1, msg);
}
@@ -715,9 +714,9 @@ public class FSEditLog implements LogsPurgeable {
final String msg =
"Could not sync enough journals to persistent storage. "
+ "Unsynced transactions: " + (txid - synctxid);
- LOG.fatal(msg, new Exception());
+ LOG.error(msg, new Exception());
synchronized(journalSetLock) {
- IOUtils.cleanup(LOG, journalSet);
+ IOUtils.cleanupWithLogger(LOG, journalSet);
}
terminate(1, msg);
}
@@ -772,7 +771,7 @@ public class FSEditLog implements LogsPurgeable {
buf.append(editLogStream.getNumSync());
buf.append(" SyncTimes(ms): ");
buf.append(journalSet.getSyncTimes());
- LOG.info(buf);
+ LOG.info(buf.toString());
}
/** Record the RPC IDs if necessary */
@@ -1711,7 +1710,7 @@ public class FSEditLog implements LogsPurgeable {
if (recovery != null) {
// If recovery mode is enabled, continue loading even if we know we
// can't load up to toAtLeastTxId.
- LOG.error(e);
+ LOG.error("Exception while selecting input streams", e);
} else {
closeAllStreams(streams);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fc2a13/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
index 6aa619f..8bc8a77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLog.java
@@ -130,7 +130,7 @@ public class TestEditLog {
/**
* A garbage mkdir op which is used for testing
- * {@link EditLogFileInputStream#scanEditLog(File)}
+ * {@link EditLogFileInputStream#scanEditLog(File, long, boolean)}
*/
public static class GarbageMkdirOp extends FSEditLogOp {
public GarbageMkdirOp() {
@@ -1141,7 +1141,7 @@ public class TestEditLog {
/**
* Construct the failure specification.
* @param roll number to fail after. e.g. 1 to fail after the first roll
- * @param loginfo index of journal to fail.
+ * @param logindex index of journal to fail.
*/
AbortSpec(int roll, int logindex) {
this.roll = roll;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fc2a13/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
index c60d79f..eb4f042 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogAutoroll.java
@@ -27,8 +27,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_CHECK_INTERVAL_MS;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_EDIT_LOG_AUTOROLL_MULTIPLIER_THRESHOLD;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -38,7 +36,9 @@ import org.apache.hadoop.hdfs.MiniDFSNNTopology;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NameNodeEditLogRoller;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +51,7 @@ import com.google.common.base.Supplier;
@RunWith(Parameterized.class)
public class TestEditLogAutoroll {
static {
- GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.DEBUG);
}
@Parameters
@@ -74,7 +74,7 @@ public class TestEditLogAutoroll {
private FSEditLog editLog;
private final Random random = new Random();
- private static final Log LOG = LogFactory.getLog(TestEditLog.class);
+ public static final Logger LOG = LoggerFactory.getLogger(FSEditLog.class);
@Before
public void setUp() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fc2a13/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
index 195ce5e..46010e0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestEditLogRace.java
@@ -52,7 +52,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Time;
-import org.apache.log4j.Level;
+import org.slf4j.event.Level;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -67,7 +67,7 @@ import org.mockito.stubbing.Answer;
@RunWith(Parameterized.class)
public class TestEditLogRace {
static {
- GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.DEBUG);
}
@Parameters
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60fc2a13/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
index b3bb3dd..9d218c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestEditLogTailer.java
@@ -60,7 +60,7 @@ import org.mockito.Mockito;
@RunWith(Parameterized.class)
public class TestEditLogTailer {
static {
- GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, org.slf4j.event.Level.DEBUG);
}
@Parameters
@@ -82,9 +82,9 @@ public class TestEditLogTailer {
static final long NN_LAG_TIMEOUT = 10 * 1000;
static {
- GenericTestUtils.setLogLevel(FSImage.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(FSEditLog.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(EditLogTailer.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(FSImage.LOG, Level.DEBUG);
+ GenericTestUtils.setLogLevel(FSEditLog.LOG, org.slf4j.event.Level.DEBUG);
+ GenericTestUtils.setLogLevel(EditLogTailer.LOG, Level.DEBUG);
}
private static Configuration getConf() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/49] hadoop git commit: YARN-7482. Max applications calculation
per queue has to be retrospected with absolute resource support. Contributed
by Sunil G.
Posted by su...@apache.org.
YARN-7482. Max applications calculation per queue has to be retrospected with absolute resource support. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c55cc516
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c55cc516
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c55cc516
Branch: refs/heads/YARN-5881
Commit: c55cc516e488171ac0d56dbad91c1b70daca8b01
Parents: a0238ca
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Thu Nov 16 17:33:00 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../scheduler/capacity/ParentQueue.java | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c55cc516/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 940637e..a427fb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1088,6 +1088,26 @@ public class ParentQueue extends AbstractCSQueue {
childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label,
(float) childQueue.getQueueCapacities().getMaximumCapacity(label)
/ getQueueCapacities().getAbsoluteMaximumCapacity(label));
+
+ // Re-visit max applications for a queue based on absolute capacity if
+ // needed.
+ if (childQueue instanceof LeafQueue) {
+ LeafQueue leafQueue = (LeafQueue) childQueue;
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ int maxApplications = (int) (conf.getMaximumSystemApplications()
+ * childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+ leafQueue.setMaxApplications(maxApplications);
+
+ int maxApplicationsPerUser = Math.min(maxApplications,
+ (int) (maxApplications
+ * (leafQueue.getUsersManager().getUserLimit() / 100.0f)
+ * leafQueue.getUsersManager().getUserLimitFactor()));
+ leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser);
+ LOG.info("LeafQueue:" + leafQueue.getQueueName() + ", maxApplications="
+ + maxApplications + ", maxApplicationsPerUser="
+ + maxApplicationsPerUser + ", Abs Cap:"
+ + childQueue.getQueueCapacities().getAbsoluteCapacity(label));
+ }
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
index da1fc5a..ef5a434 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInstrumentation.java
@@ -24,7 +24,12 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.metrics2.MetricStringBuilder;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.MetricsRecordBuilder;
+import org.apache.hadoop.metrics2.MetricsTag;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.Interns;
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
@@ -122,8 +127,23 @@ public class S3AInstrumentation {
STREAM_WRITE_BLOCK_UPLOADS_ABORTED,
STREAM_WRITE_TOTAL_TIME,
STREAM_WRITE_TOTAL_DATA,
+ COMMITTER_COMMITS_CREATED,
+ COMMITTER_COMMITS_COMPLETED,
+ COMMITTER_JOBS_SUCCEEDED,
+ COMMITTER_JOBS_FAILED,
+ COMMITTER_TASKS_SUCCEEDED,
+ COMMITTER_TASKS_FAILED,
+ COMMITTER_BYTES_COMMITTED,
+ COMMITTER_BYTES_UPLOADED,
+ COMMITTER_COMMITS_FAILED,
+ COMMITTER_COMMITS_ABORTED,
+ COMMITTER_COMMITS_REVERTED,
+ COMMITTER_MAGIC_FILES_CREATED,
S3GUARD_METADATASTORE_PUT_PATH_REQUEST,
- S3GUARD_METADATASTORE_INITIALIZATION
+ S3GUARD_METADATASTORE_INITIALIZATION,
+ S3GUARD_METADATASTORE_RETRY,
+ S3GUARD_METADATASTORE_THROTTLED,
+ STORE_IO_THROTTLED
};
@@ -179,8 +199,11 @@ public class S3AInstrumentation {
gauge(statistic.getSymbol(), statistic.getDescription());
}
//todo need a config for the quantiles interval?
+ int interval = 1;
quantiles(S3GUARD_METADATASTORE_PUT_PATH_LATENCY,
- "ops", "latency", 1);
+ "ops", "latency", interval);
+ quantiles(S3GUARD_METADATASTORE_THROTTLE_RATE,
+ "events", "frequency (Hz)", interval);
}
/**
@@ -372,7 +395,7 @@ public class S3AInstrumentation {
}
/**
- * Indicate that S3A deleted one or more file.s
+ * Indicate that S3A deleted one or more files.
* @param count number of files.
*/
public void fileDeleted(int count) {
@@ -506,6 +529,14 @@ public class S3AInstrumentation {
}
/**
+ * Create a new instance of the committer statistics.
+ * @return a new committer statistics instance
+ */
+ CommitterStatistics newCommitterStatistics() {
+ return new CommitterStatistics();
+ }
+
+ /**
* Merge in the statistics of a single input stream into
* the filesystem-wide statistics.
* @param statistics stream statistics
@@ -584,9 +615,12 @@ public class S3AInstrumentation {
/**
* The inner stream was opened.
+ * @return the previous count
*/
- public void streamOpened() {
+ public long streamOpened() {
+ long count = openOperations;
openOperations++;
+ return count;
}
/**
@@ -810,10 +844,13 @@ public class S3AInstrumentation {
}
/**
- * Note an exception in a multipart complete.
+ * Note exception in a multipart complete.
+ * @param count count of exceptions
*/
- void exceptionInMultipartComplete() {
- exceptionsInMultipartFinalize.incrementAndGet();
+ void exceptionInMultipartComplete(int count) {
+ if (count > 0) {
+ exceptionsInMultipartFinalize.addAndGet(count);
+ }
}
/**
@@ -832,6 +869,15 @@ public class S3AInstrumentation {
}
/**
+ * Data has been uploaded to be committed in a subsequent operation;
+ * to be called at the end of the write.
+ * @param size size in bytes
+ */
+ public void commitUploaded(long size) {
+ incrementCounter(COMMITTER_BYTES_UPLOADED, size);
+ }
+
+ /**
* Output stream has closed.
* Trigger merge in of all statistics not updated during operation.
*/
@@ -918,5 +964,176 @@ public class S3AInstrumentation {
public void storeClosed() {
}
+
+ /**
+ * Throttled request.
+ */
+ public void throttled() {
+ incrementCounter(S3GUARD_METADATASTORE_THROTTLED, 1);
+ addValueToQuantiles(S3GUARD_METADATASTORE_THROTTLE_RATE, 1);
+ }
+
+ /**
+ * S3Guard is retrying after a (retryable) failure.
+ */
+ public void retrying() {
+ incrementCounter(S3GUARD_METADATASTORE_RETRY, 1);
+ }
+ }
+
+ /**
+ * Instrumentation exported to S3Guard Committers.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public final class CommitterStatistics {
+
+ /** A commit has been created. */
+ public void commitCreated() {
+ incrementCounter(COMMITTER_COMMITS_CREATED, 1);
+ }
+
+ /**
+ * Data has been uploaded to be committed in a subsequent operation.
+ * @param size size in bytes
+ */
+ public void commitUploaded(long size) {
+ incrementCounter(COMMITTER_BYTES_UPLOADED, size);
+ }
+
+ /**
+ * A commit has been completed.
+ * @param size size in bytes
+ */
+ public void commitCompleted(long size) {
+ incrementCounter(COMMITTER_COMMITS_COMPLETED, 1);
+ incrementCounter(COMMITTER_BYTES_COMMITTED, size);
+ }
+
+ /** A commit has been aborted. */
+ public void commitAborted() {
+ incrementCounter(COMMITTER_COMMITS_ABORTED, 1);
+ }
+
+ public void commitReverted() {
+ incrementCounter(COMMITTER_COMMITS_REVERTED, 1);
+ }
+
+ public void commitFailed() {
+ incrementCounter(COMMITTER_COMMITS_FAILED, 1);
+ }
+
+ public void taskCompleted(boolean success) {
+ incrementCounter(
+ success ? COMMITTER_TASKS_SUCCEEDED
+ : COMMITTER_TASKS_FAILED,
+ 1);
+ }
+
+ public void jobCompleted(boolean success) {
+ incrementCounter(
+ success ? COMMITTER_JOBS_SUCCEEDED
+ : COMMITTER_JOBS_FAILED,
+ 1);
+ }
+ }
+
+ /**
+ * Copy all the metrics to a map of (name, long-value).
+ * @return a map of the metrics
+ */
+ public Map<String, Long> toMap() {
+ MetricsToMap metricBuilder = new MetricsToMap(null);
+ registry.snapshot(metricBuilder, true);
+ for (Map.Entry<String, MutableCounterLong> entry :
+ streamMetrics.entrySet()) {
+ metricBuilder.tuple(entry.getKey(), entry.getValue().value());
+ }
+ return metricBuilder.getMap();
+ }
+
+ /**
+ * Convert all metrics to a map.
+ */
+ private static class MetricsToMap extends MetricsRecordBuilder {
+ private final MetricsCollector parent;
+ private final Map<String, Long> map =
+ new HashMap<>(COUNTERS_TO_CREATE.length * 2);
+
+ MetricsToMap(MetricsCollector parent) {
+ this.parent = parent;
+ }
+
+ @Override
+ public MetricsRecordBuilder tag(MetricsInfo info, String value) {
+ return this;
+ }
+
+ @Override
+ public MetricsRecordBuilder add(MetricsTag tag) {
+ return this;
+ }
+
+ @Override
+ public MetricsRecordBuilder add(AbstractMetric metric) {
+ return this;
+ }
+
+ @Override
+ public MetricsRecordBuilder setContext(String value) {
+ return this;
+ }
+
+ @Override
+ public MetricsRecordBuilder addCounter(MetricsInfo info, int value) {
+ return tuple(info, value);
+ }
+
+ @Override
+ public MetricsRecordBuilder addCounter(MetricsInfo info, long value) {
+ return tuple(info, value);
+ }
+
+ @Override
+ public MetricsRecordBuilder addGauge(MetricsInfo info, int value) {
+ return tuple(info, value);
+ }
+
+ @Override
+ public MetricsRecordBuilder addGauge(MetricsInfo info, long value) {
+ return tuple(info, value);
+ }
+
+ public MetricsToMap tuple(MetricsInfo info, long value) {
+ return tuple(info.name(), value);
+ }
+
+ public MetricsToMap tuple(String name, long value) {
+ map.put(name, value);
+ return this;
+ }
+
+ @Override
+ public MetricsRecordBuilder addGauge(MetricsInfo info, float value) {
+ return tuple(info, (long) value);
+ }
+
+ @Override
+ public MetricsRecordBuilder addGauge(MetricsInfo info, double value) {
+ return tuple(info, (long) value);
+ }
+
+ @Override
+ public MetricsCollector parent() {
+ return parent;
+ }
+
+ /**
+ * Get the map.
+ * @return the map of metrics
+ */
+ public Map<String, Long> getMap() {
+ return map;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
new file mode 100644
index 0000000..e37a554
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ARetryPolicy.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.EOFException;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.NoRouteToHostException;
+import java.net.SocketTimeoutException;
+import java.net.UnknownHostException;
+import java.nio.file.AccessDeniedException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
+import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.InvalidRequestException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.net.ConnectTimeoutException;
+
+import static org.apache.hadoop.io.retry.RetryPolicies.*;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * The S3A request retry policy.
+ *
+ * This uses the retry options in the configuration file to determine retry
+ * count and delays for "normal" retries and separately, for throttling;
+ * the latter is best handled for longer with an exponential back-off.
+ *
+ * <ol>
+ * <li> Those exceptions considered unrecoverable (networking) are
+ * failed fast.</li>
+ * <li>All non-IOEs are failed immediately. Assumed: bugs in code,
+ * unrecoverable errors, etc</li>
+ * </ol>
+ *
+ * For non-idempotent operations, only failures due to throttling or
+ * from failures which are known to only arise prior to talking to S3
+ * are retried.
+ *
+ * The retry policy is all built around that of the normal IO exceptions,
+ * particularly those extracted from
+ * {@link S3AUtils#translateException(String, Path, AmazonClientException)}.
+ * Because the {@link #shouldRetry(Exception, int, int, boolean)} method
+ * does this translation if an {@code AmazonClientException} is processed,
+ * the policy defined for the IOEs also applies to the original exceptions.
+ *
+ * Put differently: this retry policy aims to work for handlers of the
+ * untranslated exceptions, as well as the translated ones.
+ * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html">S3 Error responses</a>
+ * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html">Amazon S3 Error Best Practices</a>
+ * @see <a href="http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/CommonErrors.html">Dynamo DB Commmon errors</a>
+ */
+public class S3ARetryPolicy implements RetryPolicy {
+
+ private final RetryPolicy retryPolicy;
+
+ /**
+ * Instantiate.
+ * @param conf configuration to read.
+ */
+ public S3ARetryPolicy(Configuration conf) {
+ Preconditions.checkArgument(conf != null, "Null configuration");
+
+ // base policy from configuration
+ RetryPolicy fixedRetries = retryUpToMaximumCountWithFixedSleep(
+ conf.getInt(RETRY_LIMIT, RETRY_LIMIT_DEFAULT),
+ conf.getTimeDuration(RETRY_INTERVAL,
+ RETRY_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS),
+ TimeUnit.MILLISECONDS);
+
+ // which is wrapped by a rejection of all non-idempotent calls except
+ // for specific failures.
+ RetryPolicy retryIdempotentCalls = new FailNonIOEs(
+ new IdempotencyRetryFilter(fixedRetries));
+
+ // and a separate policy for throttle requests, which are considered
+ // repeatable, even for non-idempotent calls, as the service
+ // rejected the call entirely
+ RetryPolicy throttlePolicy = exponentialBackoffRetry(
+ conf.getInt(RETRY_THROTTLE_LIMIT, RETRY_THROTTLE_LIMIT_DEFAULT),
+ conf.getTimeDuration(RETRY_THROTTLE_INTERVAL,
+ RETRY_THROTTLE_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS),
+ TimeUnit.MILLISECONDS);
+
+ // no retry on network and tangible API issues
+ RetryPolicy fail = RetryPolicies.TRY_ONCE_THEN_FAIL;
+
+ // client connectivity: fixed retries without care for idempotency
+ RetryPolicy connectivityFailure = fixedRetries;
+
+ // the policy map maps the exact classname; subclasses do not
+ // inherit policies.
+ Map<Class<? extends Exception>, RetryPolicy> policyMap = new HashMap<>();
+
+ // failfast exceptions which we consider unrecoverable
+ policyMap.put(UnknownHostException.class, fail);
+ policyMap.put(NoRouteToHostException.class, fail);
+ policyMap.put(InterruptedException.class, fail);
+ // note this does not pick up subclasses (like socket timeout)
+ policyMap.put(InterruptedIOException.class, fail);
+ policyMap.put(AWSRedirectException.class, fail);
+ // interesting question: should this be retried ever?
+ policyMap.put(AccessDeniedException.class, fail);
+ policyMap.put(FileNotFoundException.class, fail);
+ policyMap.put(InvalidRequestException.class, fail);
+
+ // should really be handled by resubmitting to new location;
+ // that's beyond the scope of this retry policy
+ policyMap.put(AWSRedirectException.class, fail);
+
+ // throttled requests are can be retried, always
+ policyMap.put(AWSServiceThrottledException.class, throttlePolicy);
+
+ // connectivity problems are retried without worrying about idempotency
+ policyMap.put(ConnectTimeoutException.class, connectivityFailure);
+
+ // this can be a sign of an HTTP connection breaking early.
+ // which can be reacted to by another attempt if the request was idempotent.
+ // But: could also be a sign of trying to read past the EOF on a GET,
+ // which isn't going to be recovered from
+ policyMap.put(EOFException.class, retryIdempotentCalls);
+
+ // policy on a 400/bad request still ambiguous. Given it
+ // comes and goes on test runs: try again
+ policyMap.put(AWSBadRequestException.class, connectivityFailure);
+
+ // Status 500 error code is also treated as a connectivity problem
+ policyMap.put(AWSStatus500Exception.class, connectivityFailure);
+
+ // server didn't respond.
+ policyMap.put(AWSNoResponseException.class, retryIdempotentCalls);
+
+ // other operations
+ policyMap.put(AWSClientIOException.class, retryIdempotentCalls);
+ policyMap.put(AWSServiceIOException.class, retryIdempotentCalls);
+ policyMap.put(AWSS3IOException.class, retryIdempotentCalls);
+ policyMap.put(SocketTimeoutException.class, retryIdempotentCalls);
+
+ // Dynamo DB exceptions
+ // asking for more than you should get. It's a retry but should be logged
+ // trigger sleep
+ policyMap.put(ProvisionedThroughputExceededException.class, throttlePolicy);
+
+ retryPolicy = retryByException(retryIdempotentCalls, policyMap);
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception exception,
+ int retries,
+ int failovers,
+ boolean idempotent) throws Exception {
+ Exception ex = exception;
+ if (exception instanceof AmazonClientException) {
+ // uprate the amazon client exception for the purpose of exception
+ // processing.
+ ex = S3AUtils.translateException("", "",
+ (AmazonClientException) exception);
+ }
+ return retryPolicy.shouldRetry(ex, retries, failovers, idempotent);
+ }
+
+ /**
+ * Policy which fails fast any non-idempotent call; hands off
+ * all idempotent calls to the next retry policy.
+ */
+ private static final class IdempotencyRetryFilter implements RetryPolicy {
+
+ private final RetryPolicy next;
+
+ IdempotencyRetryFilter(RetryPolicy next) {
+ this.next = next;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e,
+ int retries,
+ int failovers,
+ boolean idempotent) throws Exception {
+ return
+ idempotent ?
+ next.shouldRetry(e, retries, failovers, true)
+ : RetryAction.FAIL;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "IdempotencyRetryFilter{");
+ sb.append("next=").append(next);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /**
+ * All non-IOE exceptions are failed.
+ */
+ private static final class FailNonIOEs implements RetryPolicy {
+
+ private final RetryPolicy next;
+
+ private FailNonIOEs(RetryPolicy next) {
+ this.next = next;
+ }
+
+ @Override
+ public RetryAction shouldRetry(Exception e,
+ int retries,
+ int failovers,
+ boolean isIdempotentOrAtMostOnce) throws Exception {
+ return
+ e instanceof IOException ?
+ next.shouldRetry(e, retries, failovers, true)
+ : RetryAction.FAIL;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
index c1cf7cf..4b12667 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AStorageStatistics.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.StorageStatistics;
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.EnumMap;
@@ -35,8 +36,10 @@ import java.util.concurrent.atomic.AtomicLong;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class S3AStorageStatistics extends StorageStatistics {
- private static final Logger LOG = S3AFileSystem.LOG;
+public class S3AStorageStatistics extends StorageStatistics
+ implements Iterable<StorageStatistics.LongStatistic> {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(S3AStorageStatistics.class);
public static final String NAME = "S3AStorageStatistics";
private final Map<Statistic, AtomicLong> opsCount =
@@ -97,6 +100,11 @@ public class S3AStorageStatistics extends StorageStatistics {
}
@Override
+ public Iterator<LongStatistic> iterator() {
+ return getLongStatistics();
+ }
+
+ @Override
public Long getLong(String key) {
final Statistic type = Statistic.fromSymbol(key);
return type == null ? null : opsCount.get(type).get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 6e6f4b6..70926e6 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -18,11 +18,17 @@
package org.apache.hadoop.fs.s3a;
+import com.amazonaws.AbortedException;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
+import com.amazonaws.SdkBaseException;
import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.services.dynamodbv2.model.AmazonDynamoDBException;
+import com.amazonaws.services.dynamodbv2.model.LimitExceededException;
+import com.amazonaws.services.dynamodbv2.model.ProvisionedThroughputExceededException;
+import com.amazonaws.services.dynamodbv2.model.ResourceNotFoundException;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.S3ObjectSummary;
@@ -32,12 +38,18 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.ProviderUtils;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.FileNotFoundException;
@@ -46,12 +58,15 @@ import java.io.InterruptedIOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
+import java.net.SocketTimeoutException;
import java.net.URI;
import java.nio.file.AccessDeniedException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.concurrent.ExecutionException;
import static org.apache.hadoop.fs.s3a.Constants.*;
@@ -63,8 +78,7 @@ import static org.apache.hadoop.fs.s3a.Constants.*;
@InterfaceStability.Evolving
public final class S3AUtils {
- /** Reuse the S3AFileSystem log. */
- private static final Logger LOG = S3AFileSystem.LOG;
+ private static final Logger LOG = LoggerFactory.getLogger(S3AUtils.class);
static final String CONSTRUCTOR_EXCEPTION = "constructor exception";
static final String INSTANTIATION_EXCEPTION
= "instantiation exception";
@@ -95,6 +109,8 @@ public final class S3AUtils {
S3AEncryptionMethods.SSE_S3.getMethod()
+ " is enabled but an encryption key was set in "
+ SERVER_SIDE_ENCRYPTION_KEY;
+ private static final String EOF_MESSAGE_IN_XML_PARSER
+ = "Failed to sanitize XML document destined for handler class";
private S3AUtils() {
@@ -106,6 +122,10 @@ public final class S3AUtils {
* {@link AmazonClientException} passed in, and any status codes included
* in the operation. That is: HTTP error codes are examined and can be
* used to build a more specific response.
+ *
+ * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html">S3 Error responses</a>
+ * @see <a href="http://docs.aws.amazon.com/AmazonS3/latest/dev/ErrorBestPractices.html">Amazon S3 Error Best Practices</a>
+ * @see <a href="http://docs.aws.amazon.com/amazondynamodb/latest/APIReference/CommonErrors.html">Dynamo DB Commmon errors</a>
* @param operation operation
* @param path path operated on (must not be null)
* @param exception amazon exception raised
@@ -131,19 +151,28 @@ public final class S3AUtils {
@SuppressWarnings("ThrowableInstanceNeverThrown")
public static IOException translateException(String operation,
String path,
- AmazonClientException exception) {
+ SdkBaseException exception) {
String message = String.format("%s%s: %s",
operation,
path != null ? (" on " + path) : "",
exception);
if (!(exception instanceof AmazonServiceException)) {
- if (containsInterruptedException(exception)) {
- return (IOException)new InterruptedIOException(message)
- .initCause(exception);
+ Exception innerCause = containsInterruptedException(exception);
+ if (innerCause != null) {
+ // interrupted IO, or a socket exception underneath that class
+ return translateInterruptedException(exception, innerCause, message);
+ }
+ if (signifiesConnectionBroken(exception)) {
+ // call considered an sign of connectivity failure
+ return (EOFException)new EOFException(message).initCause(exception);
}
return new AWSClientIOException(message, exception);
} else {
-
+ if (exception instanceof AmazonDynamoDBException) {
+ // special handling for dynamo DB exceptions
+ return translateDynamoDBException(message,
+ (AmazonDynamoDBException)exception);
+ }
IOException ioe;
AmazonServiceException ase = (AmazonServiceException) exception;
// this exception is non-null if the service exception is an s3 one
@@ -151,9 +180,11 @@ public final class S3AUtils {
? (AmazonS3Exception) ase
: null;
int status = ase.getStatusCode();
+ message = message + ":" + ase.getErrorCode();
switch (status) {
case 301:
+ case 307:
if (s3Exception != null) {
if (s3Exception.getAdditionalDetails() != null &&
s3Exception.getAdditionalDetails().containsKey(ENDPOINT_KEY)) {
@@ -163,11 +194,16 @@ public final class S3AUtils {
+ "the bucket.",
s3Exception.getAdditionalDetails().get(ENDPOINT_KEY), ENDPOINT);
}
- ioe = new AWSS3IOException(message, s3Exception);
+ ioe = new AWSRedirectException(message, s3Exception);
} else {
- ioe = new AWSServiceIOException(message, ase);
+ ioe = new AWSRedirectException(message, ase);
}
break;
+
+ case 400:
+ ioe = new AWSBadRequestException(message, ase);
+ break;
+
// permissions
case 401:
case 403:
@@ -186,6 +222,25 @@ public final class S3AUtils {
// a shorter one while it is being read.
case 416:
ioe = new EOFException(message);
+ ioe.initCause(ase);
+ break;
+
+ // this has surfaced as a "no response from server" message.
+ // so rare we haven't replicated it.
+ // Treating as an idempotent proxy error.
+ case 443:
+ case 444:
+ ioe = new AWSNoResponseException(message, ase);
+ break;
+
+ // throttling
+ case 503:
+ ioe = new AWSServiceThrottledException(message, ase);
+ break;
+
+ // internal error
+ case 500:
+ ioe = new AWSStatus500Exception(message, ase);
break;
default:
@@ -226,21 +281,99 @@ public final class S3AUtils {
* Recurse down the exception loop looking for any inner details about
* an interrupted exception.
* @param thrown exception thrown
- * @return true if down the execution chain the operation was an interrupt
+ * @return the actual exception if the operation was an interrupt
*/
- static boolean containsInterruptedException(Throwable thrown) {
+ static Exception containsInterruptedException(Throwable thrown) {
if (thrown == null) {
- return false;
+ return null;
}
if (thrown instanceof InterruptedException ||
- thrown instanceof InterruptedIOException) {
- return true;
+ thrown instanceof InterruptedIOException ||
+ thrown instanceof AbortedException) {
+ return (Exception)thrown;
}
// tail recurse
return containsInterruptedException(thrown.getCause());
}
/**
+ * Handles translation of interrupted exception. This includes
+ * preserving the class of the fault for better retry logic
+ * @param exception outer exception
+ * @param innerCause inner cause (which is guaranteed to be some form
+ * of interrupted exception
+ * @param message message for the new exception.
+ * @return an IOE which can be rethrown
+ */
+ private static InterruptedIOException translateInterruptedException(
+ SdkBaseException exception,
+ final Exception innerCause,
+ String message) {
+ InterruptedIOException ioe;
+ if (innerCause instanceof SocketTimeoutException) {
+ ioe = new SocketTimeoutException(message);
+ } else {
+ String name = innerCause.getClass().getName();
+ if (name.endsWith(".ConnectTimeoutException")
+ || name.endsWith("$ConnectTimeoutException")) {
+ // TCP connection http timeout from the shaded or unshaded filenames
+ // com.amazonaws.thirdparty.apache.http.conn.ConnectTimeoutException
+ ioe = new ConnectTimeoutException(message);
+ } else {
+ // any other exception
+ ioe = new InterruptedIOException(message);
+ }
+ }
+ ioe.initCause(exception);
+ return ioe;
+ }
+
+ /**
+ * Is the exception an instance of a throttling exception. That
+ * is an AmazonServiceException with a 503 response, any
+ * exception from DynamoDB for limits exceeded, or an
+ * {@link AWSServiceThrottledException}.
+ * @param ex exception to examine
+ * @return true if it is considered a throttling exception
+ */
+ public static boolean isThrottleException(Exception ex) {
+ return ex instanceof AWSServiceThrottledException
+ || ex instanceof ProvisionedThroughputExceededException
+ || ex instanceof LimitExceededException
+ || (ex instanceof AmazonServiceException
+ && 503 == ((AmazonServiceException)ex).getStatusCode());
+ }
+
+ /**
+ * Cue that an AWS exception is likely to be an EOF Exception based
+ * on the message coming back from an XML/JSON parser. This is likely
+ * to be brittle, so only a hint.
+ * @param ex exception
+ * @return true if this is believed to be a sign the connection was broken.
+ */
+ public static boolean signifiesConnectionBroken(SdkBaseException ex) {
+ return ex.toString().contains(EOF_MESSAGE_IN_XML_PARSER);
+ }
+
+ /**
+ * Translate a DynamoDB exception into an IOException.
+ * @param message preformatted message for the exception
+ * @param ex exception
+ * @return an exception to throw.
+ */
+ public static IOException translateDynamoDBException(String message,
+ AmazonDynamoDBException ex) {
+ if (isThrottleException(ex)) {
+ return new AWSServiceThrottledException(message, ex);
+ }
+ if (ex instanceof ResourceNotFoundException) {
+ return (FileNotFoundException) new FileNotFoundException(message)
+ .initCause(ex);
+ }
+ return new AWSServiceIOException(message, ex);
+ }
+
+ /**
* Get low level details of an amazon exception for logging; multi-line.
* @param e exception
* @return string details
@@ -587,7 +720,7 @@ public final class S3AUtils {
}
/**
- * Get a long option >= the minimum allowed value, supporting memory
+ * Get a long option >= the minimum allowed value, supporting memory
* prefixes K,M,G,T,P.
* @param conf configuration
* @param key key to look up
@@ -596,7 +729,7 @@ public final class S3AUtils {
* @return the value
* @throws IllegalArgumentException if the value is below the minimum
*/
- static long longBytesOption(Configuration conf,
+ public static long longBytesOption(Configuration conf,
String key,
long defVal,
long min) {
@@ -746,6 +879,133 @@ public final class S3AUtils {
return dest;
}
+
+ /**
+ * Delete a path quietly: failures are logged at DEBUG.
+ * @param fs filesystem
+ * @param path path
+ * @param recursive recursive?
+ */
+ public static void deleteQuietly(FileSystem fs,
+ Path path,
+ boolean recursive) {
+ try {
+ fs.delete(path, recursive);
+ } catch (IOException e) {
+ LOG.debug("Failed to delete {}", path, e);
+ }
+ }
+
+ /**
+ * Delete a path: failures are logged at WARN.
+ * @param fs filesystem
+ * @param path path
+ * @param recursive recursive?
+ */
+ public static void deleteWithWarning(FileSystem fs,
+ Path path,
+ boolean recursive) {
+ try {
+ fs.delete(path, recursive);
+ } catch (IOException e) {
+ LOG.warn("Failed to delete {}", path, e);
+ }
+ }
+
+
+ /**
+ * An interface for use in lambda-expressions working with
+ * directory tree listings.
+ */
+ @FunctionalInterface
+ public interface CallOnLocatedFileStatus {
+ void call(LocatedFileStatus status) throws IOException;
+ }
+
+ /**
+ * An interface for use in lambda-expressions working with
+ * directory tree listings.
+ */
+ @FunctionalInterface
+ public interface LocatedFileStatusMap<T> {
+ T call(LocatedFileStatus status) throws IOException;
+ }
+
+ /**
+ * Apply an operation to every {@link LocatedFileStatus} in a remote
+ * iterator.
+ * @param iterator iterator from a list
+ * @param eval closure to evaluate
+ * @throws IOException anything in the closure, or iteration logic.
+ */
+ public static void applyLocatedFiles(
+ RemoteIterator<LocatedFileStatus> iterator,
+ CallOnLocatedFileStatus eval) throws IOException {
+ while (iterator.hasNext()) {
+ eval.call(iterator.next());
+ }
+ }
+
+ /**
+ * Map an operation to every {@link LocatedFileStatus} in a remote
+ * iterator, returning a list of the results.
+ * @param iterator iterator from a list
+ * @param eval closure to evaluate
+ * @throws IOException anything in the closure, or iteration logic.
+ */
+ public static <T> List<T> mapLocatedFiles(
+ RemoteIterator<LocatedFileStatus> iterator,
+ LocatedFileStatusMap<T> eval) throws IOException {
+ final List<T> results = new ArrayList<>();
+ applyLocatedFiles(iterator,
+ (s) -> results.add(eval.call(s)));
+ return results;
+ }
+
+ /**
+ * Map an operation to every {@link LocatedFileStatus} in a remote
+ * iterator, returning a list of the all results which were not empty.
+ * @param iterator iterator from a list
+ * @param eval closure to evaluate
+ * @throws IOException anything in the closure, or iteration logic.
+ */
+ public static <T> List<T> flatmapLocatedFiles(
+ RemoteIterator<LocatedFileStatus> iterator,
+ LocatedFileStatusMap<Optional<T>> eval) throws IOException {
+ final List<T> results = new ArrayList<>();
+ applyLocatedFiles(iterator,
+ (s) -> eval.call(s).map(r -> results.add(r)));
+ return results;
+ }
+
+ /**
+ * List located files and filter them as a classic listFiles(path, filter)
+ * would do.
+ * @param fileSystem filesystem
+ * @param path path to list
+ * @param recursive recursive listing?
+ * @param filter filter for the filename
+ * @return the filtered list of entries
+ * @throws IOException IO failure.
+ */
+ public static List<LocatedFileStatus> listAndFilter(FileSystem fileSystem,
+ Path path, boolean recursive, PathFilter filter) throws IOException {
+ return flatmapLocatedFiles(fileSystem.listFiles(path, recursive),
+ status -> maybe(filter.accept(status.getPath()), status));
+ }
+
+ /**
+ * Convert a value into a non-empty Optional instance if
+ * the value of {@code include} is true.
+ * @param include flag to indicate the value is to be included.
+ * @param value value to return
+ * @param <T> type of option.
+ * @return if include is false, Optional.empty. Otherwise, the value.
+ */
+ public static <T> Optional<T> maybe(boolean include, T value) {
+ return include ? Optional.of(value) : Optional.empty();
+ }
+
/**
* Patch the security credential provider information in
* {@link #CREDENTIAL_PROVIDER_PATH}
@@ -937,4 +1197,36 @@ public final class S3AUtils {
return conf.get(FS_S3A_BUCKET_PREFIX + bucket + '.' + baseKey);
}
+
+ /**
+ * Path filter which ignores any file which starts with . or _.
+ */
+ public static final PathFilter HIDDEN_FILE_FILTER = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ String name = path.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+ }
+
+ @Override
+ public String toString() {
+ return "HIDDEN_FILE_FILTER";
+ }
+ };
+
+ /**
+ * A Path filter which accepts all filenames.
+ */
+ public static final PathFilter ACCEPT_ALL = new PathFilter() {
+ @Override
+ public boolean accept(Path file) {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "ACCEPT_ALL";
+ }
+ };
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
index 6b3bd46..1a0d2c3 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ListRequest.java
@@ -66,4 +66,15 @@ public class S3ListRequest {
public ListObjectsV2Request getV2() {
return v2Request;
}
+
+ @Override
+ public String toString() {
+ if (isV1()) {
+ return String.format("List %s:/%s",
+ v1Request.getBucketName(), v1Request.getPrefix());
+ } else {
+ return String.format("List %s:/%s",
+ v2Request.getBucketName(), v2Request.getPrefix());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
index 7c73a23..d1bff8a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectAttributes.java
@@ -30,7 +30,7 @@ class S3ObjectAttributes {
private S3AEncryptionMethods serverSideEncryptionAlgorithm;
private String serverSideEncryptionKey;
- public S3ObjectAttributes(
+ S3ObjectAttributes(
String bucket,
String key,
S3AEncryptionMethods serverSideEncryptionAlgorithm,
@@ -41,19 +41,19 @@ class S3ObjectAttributes {
this.serverSideEncryptionKey = serverSideEncryptionKey;
}
- public String getBucket() {
+ String getBucket() {
return bucket;
}
- public String getKey() {
+ String getKey() {
return key;
}
- public S3AEncryptionMethods getServerSideEncryptionAlgorithm() {
+ S3AEncryptionMethods getServerSideEncryptionAlgorithm() {
return serverSideEncryptionAlgorithm;
}
- public String getServerSideEncryptionKey() {
+ String getServerSideEncryptionKey() {
return serverSideEncryptionKey;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
index 777c161..871d7c5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Statistic.java
@@ -77,6 +77,8 @@ public enum Statistic {
"Number of continued object listings made"),
OBJECT_METADATA_REQUESTS("object_metadata_requests",
"Number of requests for object metadata"),
+ OBJECT_MULTIPART_UPLOAD_INITIATED("object_multipart_initiated",
+ "Object multipart upload initiated"),
OBJECT_MULTIPART_UPLOAD_ABORTED("object_multipart_aborted",
"Object multipart upload aborted"),
OBJECT_PUT_REQUESTS("object_put_requests",
@@ -142,16 +144,62 @@ public enum Statistic {
STREAM_WRITE_QUEUE_DURATION("stream_write_queue_duration",
"Total queue duration of all block uploads"),
- // S3Guard stats
+ // S3guard committer stats
+ COMMITTER_COMMITS_CREATED(
+ "committer_commits_created",
+ "Number of files to commit created"),
+ COMMITTER_COMMITS_COMPLETED(
+ "committer_commits_completed",
+ "Number of files committed"),
+ COMMITTER_JOBS_SUCCEEDED(
+ "committer_jobs_completed",
+ "Number of successful jobs"),
+ COMMITTER_JOBS_FAILED(
+ "committer_jobs_failed",
+ "Number of failed jobs"),
+ COMMITTER_TASKS_SUCCEEDED(
+ "committer_tasks_completed",
+ "Number of successful tasks"),
+ COMMITTER_TASKS_FAILED(
+ "committer_tasks_failed",
+ "Number of failed tasks"),
+ COMMITTER_BYTES_COMMITTED(
+ "committer_bytes_committed",
+ "Amount of data committed"),
+ COMMITTER_BYTES_UPLOADED(
+ "committer_bytes_uploaded",
+ "Number of bytes uploaded duing commit operations"),
+ COMMITTER_COMMITS_FAILED(
+ "committer_commits_failed",
+ "Number of commits failed"),
+ COMMITTER_COMMITS_ABORTED(
+ "committer_commits_aborted",
+ "Number of commits aborted"),
+ COMMITTER_COMMITS_REVERTED(
+ "committer_commits_reverted",
+ "Number of commits reverted"),
+ COMMITTER_MAGIC_FILES_CREATED(
+ "committer_magic_files_created",
+ "Number of files created under 'magic' paths"),
+
+ // S3guard stats
S3GUARD_METADATASTORE_PUT_PATH_REQUEST(
"s3guard_metadatastore_put_path_request",
- "s3guard metadata store put one metadata path request"),
+ "S3Guard metadata store put one metadata path request"),
S3GUARD_METADATASTORE_PUT_PATH_LATENCY(
"s3guard_metadatastore_put_path_latency",
- "s3guard metadata store put one metadata path lantency"),
+ "S3Guard metadata store put one metadata path latency"),
S3GUARD_METADATASTORE_INITIALIZATION("s3guard_metadatastore_initialization",
- "s3guard metadata store initialization times");
+ "S3Guard metadata store initialization times"),
+ S3GUARD_METADATASTORE_RETRY("s3guard_metadatastore_retry",
+ "S3Guard metadata store retry events"),
+ S3GUARD_METADATASTORE_THROTTLED("s3guard_metadatastore_throttled",
+ "S3Guard metadata store throttled events"),
+ S3GUARD_METADATASTORE_THROTTLE_RATE(
+ "s3guard_metadatastore_throttle_rate",
+ "S3Guard metadata store throttle rate"),
+ STORE_IO_THROTTLED("store_io_throttled", "Requests throttled and retried");
private static final Map<String, Statistic> SYMBOL_MAP =
new HashMap<>(Statistic.values().length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
new file mode 100644
index 0000000..b3dd4e2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/WriteOperationHelper.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.MultipartUpload;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
+import com.amazonaws.services.s3.transfer.model.UploadResult;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.Path;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.hadoop.fs.s3a.Invoker.*;
+
+/**
+ * Helper for low-level operations against an S3 Bucket for writing data
+ * and creating and committing pending writes.
+ * <p>
+ * It hides direct access to the S3 API
+ * and is a location where the object upload process can be evolved/enhanced.
+ * <p>
+ * Features
+ * <ul>
+ * <li>Methods to create and submit requests to S3, so avoiding
+ * all direct interaction with the AWS APIs.</li>
+ * <li>Some extra preflight checks of arguments, so failing fast on
+ * errors.</li>
+ * <li>Callbacks to let the FS know of events in the output stream
+ * upload process.</li>
+ * <li>Failure handling, including converting exceptions to IOEs.</li>
+ * <li>Integration with instrumentation and S3Guard.</li>
+ * </ul>
+ *
+ * This API is for internal use only.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class WriteOperationHelper {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WriteOperationHelper.class);
+ private final S3AFileSystem owner;
+ private final Invoker invoker;
+
+ /**
+ * Constructor.
+ * @param owner owner FS creating the helper
+ *
+ */
+ protected WriteOperationHelper(S3AFileSystem owner) {
+ this.owner = owner;
+ this.invoker = new Invoker(new S3ARetryPolicy(owner.getConf()),
+ this::operationRetried);
+ }
+
+ /**
+ * Callback from {@link Invoker} when an operation is retried.
+ * @param text text of the operation
+ * @param ex exception
+ * @param retries number of retries
+ * @param idempotent is the method idempotent
+ */
+ void operationRetried(String text, Exception ex, int retries,
+ boolean idempotent) {
+ owner.operationRetried(text, ex, retries, idempotent);
+ }
+
+ /**
+ * Execute a function with retry processing.
+ * @param action action to execute (used in error messages)
+ * @param path path of work (used in error messages)
+ * @param idempotent does the operation have semantics
+ * which mean that it can be retried even if was already executed?
+ * @param operation operation to execute
+ * @param <T> type of return value
+ * @return the result of the call
+ * @throws IOException any IOE raised, or translated exception
+ */
+ public <T> T retry(String action,
+ String path,
+ boolean idempotent,
+ Invoker.Operation<T> operation)
+ throws IOException {
+
+ return invoker.retry(action, path, idempotent, operation);
+ }
+
+ /**
+ * Create a {@link PutObjectRequest} request against the specific key.
+ * @param destKey destination key
+ * @param inputStream source data.
+ * @param length size, if known. Use -1 for not known
+ * @return the request
+ */
+ public PutObjectRequest createPutObjectRequest(String destKey,
+ InputStream inputStream, long length) {
+ return owner.newPutObjectRequest(destKey,
+ newObjectMetadata(length),
+ inputStream);
+ }
+
+ /**
+ * Create a {@link PutObjectRequest} request to upload a file.
+ * @param dest key to PUT to.
+ * @param sourceFile source file
+ * @return the request
+ */
+ public PutObjectRequest createPutObjectRequest(String dest,
+ File sourceFile) {
+ Preconditions.checkState(sourceFile.length() < Integer.MAX_VALUE,
+ "File length is too big for a single PUT upload");
+ return owner.newPutObjectRequest(dest,
+ newObjectMetadata((int) sourceFile.length()),
+ sourceFile);
+ }
+
+ /**
+ * Callback on a successful write.
+ * @param length length of the write
+ */
+ public void writeSuccessful(long length) {
+ }
+
+ /**
+ * Callback on a write failure.
+ * @param ex Any exception raised which triggered the failure.
+ */
+ public void writeFailed(Exception ex) {
+ LOG.debug("Write to {} failed", this, ex);
+ }
+
+ /**
+ * Create a new object metadata instance.
+ * Any standard metadata headers are added here, for example:
+ * encryption.
+ * @param length size, if known. Use -1 for not known
+ * @return a new metadata instance
+ */
+ public ObjectMetadata newObjectMetadata(long length) {
+ return owner.newObjectMetadata(length);
+ }
+
+ /**
+ * Start the multipart upload process.
+ * Retry policy: retrying, translated.
+ * @return the upload result containing the ID
+ * @throws IOException IO problem
+ */
+ @Retries.RetryTranslated
+ public String initiateMultiPartUpload(String destKey) throws IOException {
+ LOG.debug("Initiating Multipart upload to {}", destKey);
+ final InitiateMultipartUploadRequest initiateMPURequest =
+ new InitiateMultipartUploadRequest(owner.getBucket(),
+ destKey,
+ newObjectMetadata(-1));
+ initiateMPURequest.setCannedACL(owner.getCannedACL());
+ owner.setOptionalMultipartUploadRequestParameters(initiateMPURequest);
+
+ return retry("initiate MultiPartUpload", destKey, true,
+ () -> owner.initiateMultipartUpload(initiateMPURequest).getUploadId());
+ }
+
+ /**
+ * Finalize a multipart PUT operation.
+ * This completes the upload, and, if that works, calls
+ * {@link S3AFileSystem#finishedWrite(String, long)} to update the filesystem.
+ * Retry policy: retrying, translated.
+ * @param destKey destination of the commit
+ * @param uploadId multipart operation Id
+ * @param partETags list of partial uploads
+ * @param length length of the upload
+ * @param retrying retrying callback
+ * @return the result of the operation.
+ * @throws IOException on problems.
+ */
+ @Retries.RetryTranslated
+ private CompleteMultipartUploadResult finalizeMultipartUpload(
+ String destKey,
+ String uploadId,
+ List<PartETag> partETags,
+ long length,
+ Retried retrying) throws IOException {
+ return invoker.retry("Completing multipart commit", destKey,
+ true,
+ retrying,
+ () -> {
+ // a copy of the list is required, so that the AWS SDK doesn't
+ // attempt to sort an unmodifiable list.
+ CompleteMultipartUploadResult result =
+ owner.getAmazonS3Client().completeMultipartUpload(
+ new CompleteMultipartUploadRequest(owner.getBucket(),
+ destKey,
+ uploadId,
+ new ArrayList<>(partETags)));
+ owner.finishedWrite(destKey, length);
+ return result;
+ }
+ );
+ }
+
+ /**
+ * This completes a multipart upload to the destination key via
+ * {@code finalizeMultipartUpload()}.
+ * Retry policy: retrying, translated.
+ * Retries increment the {@code errorCount} counter.
+ * @param destKey destination
+ * @param uploadId multipart operation Id
+ * @param partETags list of partial uploads
+ * @param length length of the upload
+ * @param errorCount a counter incremented by 1 on every error; for
+ * use in statistics
+ * @return the result of the operation.
+ * @throws IOException if problems arose which could not be retried, or
+ * the retry count was exceeded
+ */
+ @Retries.RetryTranslated
+ public CompleteMultipartUploadResult completeMPUwithRetries(
+ String destKey,
+ String uploadId,
+ List<PartETag> partETags,
+ long length,
+ AtomicInteger errorCount)
+ throws IOException {
+ checkNotNull(uploadId);
+ checkNotNull(partETags);
+ LOG.debug("Completing multipart upload {} with {} parts",
+ uploadId, partETags.size());
+ return finalizeMultipartUpload(destKey,
+ uploadId,
+ partETags,
+ length,
+ (text, e, r, i) -> errorCount.incrementAndGet());
+ }
+
+ /**
+ * Abort a multipart upload operation.
+ * @param destKey destination key of the upload
+ * @param uploadId multipart operation Id
+ * @param retrying callback invoked on every retry
+ * @throws IOException failure to abort
+ * @throws FileNotFoundException if the abort ID is unknown
+ */
+ @Retries.RetryTranslated
+ public void abortMultipartUpload(String destKey, String uploadId,
+ Retried retrying)
+ throws IOException {
+ invoker.retry("Aborting multipart upload", destKey, true,
+ retrying,
+ () -> owner.abortMultipartUpload(
+ destKey,
+ uploadId));
+ }
+
+ /**
+ * Abort a multipart commit operation.
+ * @param upload upload to abort.
+ * @throws IOException on problems.
+ */
+ @Retries.RetryTranslated
+ public void abortMultipartUpload(MultipartUpload upload)
+ throws IOException {
+ invoker.retry("Aborting multipart commit", upload.getKey(), true,
+ () -> owner.abortMultipartUpload(upload));
+ }
+
+
+ /**
+ * Abort multipart uploads under a path: limited to the first
+ * few hundred.
+ * @param prefix prefix for uploads to abort
+ * @return a count of aborts
+ * @throws IOException trouble; FileNotFoundExceptions are swallowed.
+ */
+ @Retries.RetryTranslated
+ public int abortMultipartUploadsUnderPath(String prefix)
+ throws IOException {
+ LOG.debug("Aborting multipart uploads under {}", prefix);
+ int count = 0;
+ List<MultipartUpload> multipartUploads = owner.listMultipartUploads(prefix);
+ LOG.debug("Number of outstanding uploads: {}", multipartUploads.size());
+ for (MultipartUpload upload: multipartUploads) {
+ try {
+ abortMultipartUpload(upload);
+ count++;
+ } catch (FileNotFoundException e) {
+ LOG.debug("Already aborted: {}", upload.getKey(), e);
+ }
+ }
+ return count;
+ }
+
+ /**
+ * Abort a multipart commit operation.
+ * @param destKey destination key of ongoing operation
+ * @param uploadId multipart operation Id
+ * @throws IOException on problems.
+ * @throws FileNotFoundException if the abort ID is unknown
+ */
+ @Retries.RetryTranslated
+ public void abortMultipartCommit(String destKey, String uploadId)
+ throws IOException {
+ abortMultipartUpload(destKey, uploadId, invoker.getRetryCallback());
+ }
+
+ /**
+ * Create and initialize a part request of a multipart upload.
+ * Exactly one of: {@code uploadStream} or {@code sourceFile}
+ * must be specified.
+ * A subset of the file may be posted, by providing the starting point
+ * in {@code offset} and a length of block in {@code size} equal to
+ * or less than the remaining bytes.
+ * @param destKey destination key of ongoing operation
+ * @param uploadId ID of ongoing upload
+ * @param partNumber current part number of the upload
+ * @param size amount of data
+ * @param uploadStream source of data to upload
+ * @param sourceFile optional source file.
+ * @param offset offset in file to start reading.
+ * @return the request.
+ */
+ public UploadPartRequest newUploadPartRequest(
+ String destKey,
+ String uploadId,
+ int partNumber,
+ int size,
+ InputStream uploadStream,
+ File sourceFile,
+ Long offset) {
+ checkNotNull(uploadId);
+ // exactly one source must be set; xor verifies this
+ checkArgument((uploadStream != null) ^ (sourceFile != null),
+ "Data source");
+ checkArgument(size >= 0, "Invalid partition size %s", size);
+ checkArgument(partNumber > 0 && partNumber <= 10000,
+ "partNumber must be between 1 and 10000 inclusive, but is %s",
+ partNumber);
+
+ LOG.debug("Creating part upload request for {} #{} size {}",
+ uploadId, partNumber, size);
+ UploadPartRequest request = new UploadPartRequest()
+ .withBucketName(owner.getBucket())
+ .withKey(destKey)
+ .withUploadId(uploadId)
+ .withPartNumber(partNumber)
+ .withPartSize(size);
+ if (uploadStream != null) {
+ // there's an upload stream. Bind to it.
+ request.setInputStream(uploadStream);
+ } else {
+ checkArgument(sourceFile.exists(),
+ "Source file does not exist: %s", sourceFile);
+ checkArgument(offset >= 0, "Invalid offset %s", offset);
+ long length = sourceFile.length();
+ checkArgument(offset == 0 || offset < length,
+ "Offset %s beyond length of file %s", offset, length);
+ request.setFile(sourceFile);
+ request.setFileOffset(offset);
+ }
+ return request;
+ }
+
+ /**
+ * The toString method is intended to be used in logging/toString calls.
+ * @return a string description.
+ */
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "WriteOperationHelper {bucket=").append(owner.getBucket());
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * PUT an object directly (i.e. not via the transfer manager).
+ * Byte length is calculated from the file length, or, if there is no
+ * file, from the content length of the header.
+ * @param putObjectRequest the request
+ * @return the upload initiated
+ * @throws IOException on problems
+ */
+ @Retries.RetryTranslated
+ public PutObjectResult putObject(PutObjectRequest putObjectRequest)
+ throws IOException {
+ return retry("put",
+ putObjectRequest.getKey(), true,
+ () -> owner.putObjectDirect(putObjectRequest));
+ }
+
+ /**
+ * PUT an object via the transfer manager.
+ * @param putObjectRequest the request
+ * @return the result of the operation
+ * @throws IOException on problems
+ */
+ @Retries.OnceTranslated
+ public UploadResult uploadObject(PutObjectRequest putObjectRequest)
+ throws IOException {
+ // no retry; rely on xfer manager logic
+ return retry("put",
+ putObjectRequest.getKey(), true,
+ () -> owner.executePut(putObjectRequest, null));
+ }
+
+ /**
+ * Revert a commit by deleting the file.
+ * Relies on retry code in filesystem
+ * @throws IOException on problems
+ * @param destKey destination key
+ */
+ @Retries.RetryTranslated
+ public void revertCommit(String destKey) throws IOException {
+ once("revert commit", destKey,
+ () -> {
+ Path destPath = owner.keyToQualifiedPath(destKey);
+ owner.deleteObjectAtPath(destPath,
+ destKey, true);
+ owner.maybeCreateFakeParentDirectory(destPath);
+ }
+ );
+ }
+
+ /**
+ * Upload part of a multi-partition file.
+ * @param request request
+ * @return the result of the operation.
+ * @throws IOException on problems
+ */
+ @Retries.RetryTranslated
+ public UploadPartResult uploadPart(UploadPartRequest request)
+ throws IOException {
+ return retry("upload part",
+ request.getKey(),
+ true,
+ () -> owner.uploadPart(request));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
new file mode 100644
index 0000000..3277916
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitter.java
@@ -0,0 +1,756 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import com.amazonaws.services.s3.model.MultipartUpload;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Invoker;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.net.NetUtils;
+
+import static org.apache.hadoop.fs.s3a.Invoker.ignoreIOExceptions;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitUtilsWithMR.*;
+
+/**
+ * Abstract base class for S3A committers; allows for any commonality
+ * between different architectures.
+ *
+ * Although the committer APIs allow for a committer to be created without
+ * an output path, this is not supported in this class or its subclasses:
+ * a destination must be supplied. It is left to the committer factory
+ * to handle the creation of a committer when the destination is unknown.
+ *
+ * Requiring an output directory simplifies coding and testing.
+ */
+public abstract class AbstractS3ACommitter extends PathOutputCommitter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AbstractS3ACommitter.class);
+
+ /**
+ * Thread pool for task execution.
+ */
+ private ExecutorService threadPool;
+
+ /** Underlying commit operations. */
+ private final CommitOperations commitOperations;
+
+ /**
+ * Final destination of work.
+ */
+ private Path outputPath;
+
+ /**
+ * Role: used in log/text messages.
+ */
+ private final String role;
+
+ /**
+ * This is the directory for all intermediate work: where the output format
+ * will write data.
+ * <i>This may not be on the final file system</i>
+ */
+ private Path workPath;
+
+ /** Configuration of the job. */
+ private Configuration conf;
+
+ /** Filesystem of {@link #outputPath}. */
+ private FileSystem destFS;
+
+ /** The job context. For a task, this can be cast to a TaskContext. */
+ private final JobContext jobContext;
+
+ /** Should a job marker be created? */
+ private final boolean createJobMarker;
+
+ /**
+ * Create a committer.
+ * This constructor binds the destination directory and configuration, but
+ * does not update the work path: That must be calculated by the
+ * implementation;
+ * It is omitted here to avoid subclass methods being called too early.
+ * @param outputPath the job's output path: MUST NOT be null.
+ * @param context the task's context
+ * @throws IOException on a failure
+ */
+ protected AbstractS3ACommitter(
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ Preconditions.checkArgument(outputPath != null, "null output path");
+ Preconditions.checkArgument(context != null, "null job context");
+ this.jobContext = context;
+ this.role = "Task committer " + context.getTaskAttemptID();
+ setConf(context.getConfiguration());
+ initOutput(outputPath);
+ LOG.debug("{} instantiated for job \"{}\" ID {} with destination {}",
+ role, jobName(context), jobIdString(context), outputPath);
+ S3AFileSystem fs = getDestS3AFS();
+ createJobMarker = context.getConfiguration().getBoolean(
+ CREATE_SUCCESSFUL_JOB_OUTPUT_DIR_MARKER,
+ DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER);
+ commitOperations = new CommitOperations(fs);
+ }
+
+ /**
+ * Init the output filesystem and path.
+ * TESTING ONLY; allows mock FS to cheat.
+ * @param out output path
+ * @throws IOException failure to create the FS.
+ */
+ @VisibleForTesting
+ protected void initOutput(Path out) throws IOException {
+ FileSystem fs = getDestinationFS(out, getConf());
+ setDestFS(fs);
+ setOutputPath(fs.makeQualified(out));
+ }
+
+ /**
+ * Get the job/task context this committer was instantiated with.
+ * @return the context.
+ */
+ public final JobContext getJobContext() {
+ return jobContext;
+ }
+
+ /**
+ * Final path of output, in the destination FS.
+ * @return the path
+ */
+ @Override
+ public final Path getOutputPath() {
+ return outputPath;
+ }
+
+ /**
+ * Set the output path.
+ * @param outputPath new value
+ */
+ protected final void setOutputPath(Path outputPath) {
+ Preconditions.checkNotNull(outputPath, "Null output path");
+ this.outputPath = outputPath;
+ }
+
+ /**
+ * This is the critical method for {@code FileOutputFormat}; it declares
+ * the path for work.
+ * @return the working path.
+ */
+ @Override
+ public Path getWorkPath() {
+ return workPath;
+ }
+
+ /**
+ * Set the work path for this committer.
+ * @param workPath the work path to use.
+ */
+ protected void setWorkPath(Path workPath) {
+ LOG.debug("Setting work path to {}", workPath);
+ this.workPath = workPath;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ protected void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ /**
+ * Get the destination FS, creating it on demand if needed.
+ * @return the filesystem; requires the output path to be set up
+ * @throws IOException if the FS cannot be instantiated.
+ */
+ public FileSystem getDestFS() throws IOException {
+ if (destFS == null) {
+ FileSystem fs = getDestinationFS(outputPath, getConf());
+ setDestFS(fs);
+ }
+ return destFS;
+ }
+
+ /**
+ * Get the destination as an S3A Filesystem; casting it.
+ * @return the dest S3A FS.
+ * @throws IOException if the FS cannot be instantiated.
+ */
+ public S3AFileSystem getDestS3AFS() throws IOException {
+ return (S3AFileSystem) getDestFS();
+ }
+
+ /**
+ * Set the destination FS: the FS of the final output.
+ * @param destFS destination FS.
+ */
+ protected void setDestFS(FileSystem destFS) {
+ this.destFS = destFS;
+ }
+
+ /**
+ * Compute the path where the output of a given job attempt will be placed.
+ * @param context the context of the job. This is used to get the
+ * application attempt ID.
+ * @return the path to store job attempt data.
+ */
+ public Path getJobAttemptPath(JobContext context) {
+ return getJobAttemptPath(getAppAttemptId(context));
+ }
+
+ /**
+ * Compute the path where the output of a given job attempt will be placed.
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @return the path to store job attempt data.
+ */
+ protected abstract Path getJobAttemptPath(int appAttemptId);
+
+ /**
+ * Compute the path where the output of a task attempt is stored until
+ * that task is committed. This may be the normal Task attempt path
+ * or it may be a subdirectory.
+ * The default implementation returns the value of
+ * {@link #getBaseTaskAttemptPath(TaskAttemptContext)};
+ * subclasses may return different values.
+ * @param context the context of the task attempt.
+ * @return the path where a task attempt should be stored.
+ */
+ public Path getTaskAttemptPath(TaskAttemptContext context) {
+ return getBaseTaskAttemptPath(context);
+ }
+
+ /**
+ * Compute the base path where the output of a task attempt is written.
+ * This is the path which will be deleted when a task is cleaned up and
+ * aborted.
+ *
+ * @param context the context of the task attempt.
+ * @return the path where a task attempt should be stored.
+ */
+ protected abstract Path getBaseTaskAttemptPath(TaskAttemptContext context);
+
+ /**
+ * Get a temporary directory for data. When a task is aborted/cleaned
+ * up, the contents of this directory are all deleted.
+ * @param context task context
+ * @return a path for temporary data.
+ */
+ public abstract Path getTempTaskAttemptPath(TaskAttemptContext context);
+
+ /**
+ * Get the name of this committer.
+ * @return the committer name.
+ */
+ public abstract String getName();
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "AbstractS3ACommitter{");
+ sb.append("role=").append(role);
+ sb.append(", name").append(getName());
+ sb.append(", outputPath=").append(getOutputPath());
+ sb.append(", workPath=").append(workPath);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Get the destination filesystem from the output path and the configuration.
+ * @param out output path
+ * @param config job/task config
+ * @return the associated FS
+ * @throws PathCommitException output path isn't to an S3A FS instance.
+ * @throws IOException failure to instantiate the FS.
+ */
+ protected FileSystem getDestinationFS(Path out, Configuration config)
+ throws IOException {
+ return getS3AFileSystem(out, config,
+ requiresDelayedCommitOutputInFileSystem());
+ }
+
+ /**
+ * Flag to indicate whether or not the destination filesystem needs
+ * to be configured to support magic paths where the output isn't immediately
+ * visible. If the committer returns true, then committer setup will
+ * fail if the FS doesn't have the capability.
+ * Base implementation returns false.
+ * @return what the requirements of the committer are of the filesystem.
+ */
+ protected boolean requiresDelayedCommitOutputInFileSystem() {
+ return false;
+ }
+ /**
+ * Task recovery considered unsupported: Warn and fail.
+ * @param taskContext Context of the task whose output is being recovered
+ * @throws IOException always.
+ */
+ @Override
+ public void recoverTask(TaskAttemptContext taskContext) throws IOException {
+ LOG.warn("Cannot recover task {}", taskContext.getTaskAttemptID());
+ throw new PathCommitException(outputPath,
+ String.format("Unable to recover task %s",
+ taskContext.getTaskAttemptID()));
+ }
+
+ /**
+ * if the job requires a success marker on a successful job,
+ * create the file {@link CommitConstants#_SUCCESS}.
+ *
+ * While the classic committers create a 0-byte file, the S3Guard committers
+ * PUT up a the contents of a {@link SuccessData} file.
+ * @param context job context
+ * @param pending the pending commits
+ * @throws IOException IO failure
+ */
+ protected void maybeCreateSuccessMarkerFromCommits(JobContext context,
+ List<SinglePendingCommit> pending) throws IOException {
+ List<String> filenames = new ArrayList<>(pending.size());
+ for (SinglePendingCommit commit : pending) {
+ String key = commit.getDestinationKey();
+ if (!key.startsWith("/")) {
+ // fix up so that FS.makeQualified() sets up the path OK
+ key = "/" + key;
+ }
+ filenames.add(key);
+ }
+ maybeCreateSuccessMarker(context, filenames);
+ }
+
+ /**
+ * if the job requires a success marker on a successful job,
+ * create the file {@link CommitConstants#_SUCCESS}.
+ *
+ * While the classic committers create a 0-byte file, the S3Guard committers
+ * PUT up a the contents of a {@link SuccessData} file.
+ * @param context job context
+ * @param filenames list of filenames.
+ * @throws IOException IO failure
+ */
+ protected void maybeCreateSuccessMarker(JobContext context,
+ List<String> filenames)
+ throws IOException {
+ if (createJobMarker) {
+ // create a success data structure and then save it
+ SuccessData successData = new SuccessData();
+ successData.setCommitter(getName());
+ successData.setDescription(getRole());
+ successData.setHostname(NetUtils.getLocalHostname());
+ Date now = new Date();
+ successData.setTimestamp(now.getTime());
+ successData.setDate(now.toString());
+ successData.setFilenames(filenames);
+ commitOperations.createSuccessMarker(getOutputPath(), successData, true);
+ }
+ }
+
+ /**
+ * Base job setup deletes the success marker.
+ * TODO: Do we need this?
+ * @param context context
+ * @throws IOException IO failure
+ */
+/*
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ if (createJobMarker) {
+ try (DurationInfo d = new DurationInfo("Deleting _SUCCESS marker")) {
+ commitOperations.deleteSuccessMarker(getOutputPath());
+ }
+ }
+ }
+*/
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG, "Setup Task %s",
+ context.getTaskAttemptID())) {
+ Path taskAttemptPath = getTaskAttemptPath(context);
+ FileSystem fs = getTaskAttemptFilesystem(context);
+ fs.mkdirs(taskAttemptPath);
+ }
+ }
+
+ /**
+ * Get the task attempt path filesystem. This may not be the same as the
+ * final destination FS, and so may not be an S3A FS.
+ * @param context task attempt
+ * @return the filesystem
+ * @throws IOException failure to instantiate
+ */
+ protected FileSystem getTaskAttemptFilesystem(TaskAttemptContext context)
+ throws IOException {
+ return getTaskAttemptPath(context).getFileSystem(getConf());
+ }
+
+ /**
+ * Commit a list of pending uploads.
+ * @param context job context
+ * @param pending list of pending uploads
+ * @throws IOException on any failure
+ */
+ protected void commitPendingUploads(JobContext context,
+ List<SinglePendingCommit> pending) throws IOException {
+ if (pending.isEmpty()) {
+ LOG.warn("{}: No pending uploads to commit", getRole());
+ }
+ LOG.debug("{}: committing the output of {} task(s)",
+ getRole(), pending.size());
+ Tasks.foreach(pending)
+ .stopOnFailure()
+ .executeWith(buildThreadPool(context))
+ .onFailure((commit, exception) ->
+ getCommitOperations().abortSingleCommit(commit))
+ .abortWith(commit -> getCommitOperations().abortSingleCommit(commit))
+ .revertWith(commit -> getCommitOperations().revertCommit(commit))
+ .run(commit -> getCommitOperations().commitOrFail(commit));
+ }
+
+ /**
+ * Try to read every pendingset file and build a list of them/
+ * In the case of a failure to read the file, exceptions are held until all
+ * reads have been attempted.
+ * @param context job context
+ * @param suppressExceptions whether to suppress exceptions.
+ * @param fs job attempt fs
+ * @param pendingCommitFiles list of files found in the listing scan
+ * @return the list of commits
+ * @throws IOException on a failure when suppressExceptions is false.
+ */
+ protected List<SinglePendingCommit> loadPendingsetFiles(
+ JobContext context,
+ boolean suppressExceptions,
+ FileSystem fs,
+ Iterable<? extends FileStatus> pendingCommitFiles) throws IOException {
+
+ final List<SinglePendingCommit> pending = Collections.synchronizedList(
+ Lists.newArrayList());
+ Tasks.foreach(pendingCommitFiles)
+ .suppressExceptions(suppressExceptions)
+ .executeWith(buildThreadPool(context))
+ .run(pendingCommitFile ->
+ pending.addAll(
+ PendingSet.load(fs, pendingCommitFile.getPath()).getCommits())
+ );
+ return pending;
+ }
+
+ /**
+ * Internal Job commit operation: where the S3 requests are made
+ * (potentially in parallel).
+ * @param context job context
+ * @param pending pending request
+ * @throws IOException any failure
+ */
+ protected void commitJobInternal(JobContext context,
+ List<SinglePendingCommit> pending)
+ throws IOException {
+
+ commitPendingUploads(context, pending);
+ }
+
+ @Override
+ public void abortJob(JobContext context, JobStatus.State state)
+ throws IOException {
+ LOG.info("{}: aborting job {} in state {}",
+ getRole(), jobIdString(context), state);
+ // final cleanup operations
+ abortJobInternal(context, false);
+ }
+
+
+ /**
+ * The internal job abort operation; can be overridden in tests.
+ * This must clean up operations; it is called when a commit fails, as
+ * well as in an {@link #abortJob(JobContext, JobStatus.State)} call.
+ * The base implementation calls {@link #cleanup(JobContext, boolean)}
+ * @param context job context
+ * @param suppressExceptions should exceptions be suppressed?
+ * @throws IOException any IO problem raised when suppressExceptions is false.
+ */
+ protected void abortJobInternal(JobContext context,
+ boolean suppressExceptions)
+ throws IOException {
+ cleanup(context, suppressExceptions);
+ }
+
+ /**
+ * Abort all pending uploads to the destination directory during
+ * job cleanup operations.
+ * @param suppressExceptions should exceptions be suppressed
+ */
+ protected void abortPendingUploadsInCleanup(
+ boolean suppressExceptions) throws IOException {
+ Path dest = getOutputPath();
+ try (DurationInfo d =
+ new DurationInfo(LOG, "Aborting all pending commits under %s",
+ dest)) {
+ CommitOperations ops = getCommitOperations();
+ List<MultipartUpload> pending = ops
+ .listPendingUploadsUnderPath(dest);
+ Tasks.foreach(pending)
+ .executeWith(buildThreadPool(getJobContext()))
+ .suppressExceptions(suppressExceptions)
+ .run(u -> ops.abortMultipartCommit(u.getKey(), u.getUploadId()));
+ }
+ }
+
+ /**
+ * Subclass-specific pre commit actions.
+ * @param context job context
+ * @param pending the pending operations
+ * @throws IOException any failure
+ */
+ protected void preCommitJob(JobContext context,
+ List<SinglePendingCommit> pending) throws IOException {
+ }
+
+ /**
+ * Commit work.
+ * This consists of two stages: precommit and commit.
+ * <p>
+ * Precommit: identify pending uploads, then allow subclasses
+ * to validate the state of the destination and the pending uploads.
+ * Any failure here triggers an abort of all pending uploads.
+ * <p>
+ * Commit internal: do the final commit sequence.
+ * <p>
+ * The final commit action is to build the {@code __SUCCESS} file entry.
+ * </p>
+ * @param context job context
+ * @throws IOException any failure
+ */
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ String id = jobIdString(context);
+ try (DurationInfo d = new DurationInfo(LOG,
+ "%s: commitJob(%s)", getRole(), id)) {
+ List<SinglePendingCommit> pending
+ = listPendingUploadsToCommit(context);
+ preCommitJob(context, pending);
+ commitJobInternal(context, pending);
+ jobCompleted(true);
+ maybeCreateSuccessMarkerFromCommits(context, pending);
+ cleanup(context, false);
+ } catch (IOException e) {
+ LOG.warn("Commit failure for job {}", id, e);
+ jobCompleted(false);
+ abortJobInternal(context, true);
+ throw e;
+ }
+ }
+
+ /**
+ * Job completion outcome; this may be subclassed in tests.
+ * @param success did the job succeed.
+ */
+ protected void jobCompleted(boolean success) {
+ getCommitOperations().jobCompleted(success);
+ }
+
+ /**
+ * Clean up any staging directories.
+ * IOEs must be caught and swallowed.
+ */
+ public abstract void cleanupStagingDirs();
+
+ /**
+ * Get the list of pending uploads for this job attempt.
+ * @param context job context
+ * @return a list of pending uploads.
+ * @throws IOException Any IO failure
+ */
+ protected abstract List<SinglePendingCommit> listPendingUploadsToCommit(
+ JobContext context)
+ throws IOException;
+
+ /**
+ * Cleanup the job context, including aborting anything pending.
+ * @param context job context
+ * @param suppressExceptions should exceptions be suppressed?
+ * @throws IOException any failure if exceptions were not suppressed.
+ */
+ protected void cleanup(JobContext context,
+ boolean suppressExceptions) throws IOException {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Cleanup job %s", jobIdString(context))) {
+ abortPendingUploadsInCleanup(suppressExceptions);
+ } finally {
+ cleanupStagingDirs();
+ }
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void cleanupJob(JobContext context) throws IOException {
+ String r = getRole();
+ String id = jobIdString(context);
+ LOG.warn("{}: using deprecated cleanupJob call for {}", r, id);
+ try (DurationInfo d = new DurationInfo(LOG, "%s: cleanup Job %s", r, id)) {
+ cleanup(context, true);
+ }
+ }
+
+ /**
+ * Execute an operation; maybe suppress any raised IOException.
+ * @param suppress should raised IOEs be suppressed?
+ * @param action action (for logging when the IOE is supressed.
+ * @param operation operation
+ * @throws IOException if operation raised an IOE and suppress == false
+ */
+ protected void maybeIgnore(
+ boolean suppress,
+ String action,
+ Invoker.VoidOperation operation) throws IOException {
+ if (suppress) {
+ ignoreIOExceptions(LOG, action, "", operation);
+ } else {
+ operation.execute();
+ }
+ }
+
+ /**
+ * Execute an operation; maybe suppress any raised IOException.
+ * @param suppress should raised IOEs be suppressed?
+ * @param action action (for logging when the IOE is suppressed.
+ * @param ex exception
+ * @throws IOException if suppress == false
+ */
+ protected void maybeIgnore(
+ boolean suppress,
+ String action,
+ IOException ex) throws IOException {
+ if (suppress) {
+ LOG.info(action, ex);
+ } else {
+ throw ex;
+ }
+ }
+
+ /**
+ * Get the commit actions instance.
+ * Subclasses may provide a mock version of this.
+ * @return the commit actions instance to use for operations.
+ */
+ protected CommitOperations getCommitOperations() {
+ return commitOperations;
+ }
+
+ /**
+ * Used in logging and reporting to help disentangle messages.
+ * @return the committer's role.
+ */
+ protected String getRole() {
+ return role;
+ }
+
+ /**
+ * Returns an {@link ExecutorService} for parallel tasks. The number of
+ * threads in the thread-pool is set by s3.multipart.committer.num-threads.
+ * If num-threads is 0, this will return null;
+ *
+ * @param context the JobContext for this commit
+ * @return an {@link ExecutorService} or null for the number of threads
+ */
+ protected final synchronized ExecutorService buildThreadPool(
+ JobContext context) {
+
+ if (threadPool == null) {
+ int numThreads = context.getConfiguration().getInt(
+ FS_S3A_COMMITTER_THREADS,
+ DEFAULT_COMMITTER_THREADS);
+ LOG.debug("{}: creating thread pool of size {}", getRole(), numThreads);
+ if (numThreads > 0) {
+ threadPool = Executors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("s3-committer-pool-%d")
+ .build());
+ } else {
+ return null;
+ }
+ }
+ return threadPool;
+ }
+
+ /**
+ * Delete the task attempt path without raising any errors.
+ * @param context task context
+ */
+ protected void deleteTaskAttemptPathQuietly(TaskAttemptContext context) {
+ Path attemptPath = getBaseTaskAttemptPath(context);
+ ignoreIOExceptions(LOG, "Delete task attempt path", attemptPath.toString(),
+ () -> deleteQuietly(
+ getTaskAttemptFilesystem(context), attemptPath, true));
+ }
+
+ /**
+ * Abort all pending uploads in the list.
+ * @param context job context
+ * @param pending pending uploads
+ * @param suppressExceptions should exceptions be suppressed
+ * @throws IOException any exception raised
+ */
+ protected void abortPendingUploads(JobContext context,
+ List<SinglePendingCommit> pending,
+ boolean suppressExceptions)
+ throws IOException {
+ if (pending == null || pending.isEmpty()) {
+ LOG.info("{}: no pending commits to abort", getRole());
+ } else {
+ try (DurationInfo d = new DurationInfo(LOG,
+ "Aborting %s uploads", pending.size())) {
+ Tasks.foreach(pending)
+ .executeWith(buildThreadPool(context))
+ .suppressExceptions(suppressExceptions)
+ .run(commit -> getCommitOperations().abortSingleCommit(commit));
+ }
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/49] hadoop git commit: YARN-7525. Incorrect query parameters in
cluster nodes REST API document. Contributed by Tao Yang.
Posted by su...@apache.org.
YARN-7525. Incorrect query parameters in cluster nodes REST API document. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96e6a993
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96e6a993
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96e6a993
Branch: refs/heads/YARN-5881
Commit: 96e6a993609d170b4d2c70b2dba8b4530093606f
Parents: d5f6688
Author: bibinchundatt <bi...@apache.org>
Authored: Sat Nov 18 19:02:11 2017 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Sat Nov 18 19:02:11 2017 +0530
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96e6a993/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index f8048a8..f478403 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -2107,8 +2107,7 @@ With the Nodes API, you can obtain a collection of resources, each of which repr
### Query Parameters Supported
- * state - the state of the node
- * healthy - true or false
+ * states - the states of the node, specified as a comma-separated list, valid values are: NEW, RUNNING, UNHEALTHY, DECOMMISSIONING, DECOMMISSIONED, LOST, REBOOTED, SHUTDOWN
### Elements of the *nodes* object
@@ -2286,7 +2285,7 @@ Use the following URI to obtain a Node Object, from a node identified by the nod
| Item | Data Type | Description |
|:---- |:---- |:---- |
| rack | string | The rack location of this node |
-| state | string | State of the node - valid values are: NEW, RUNNING, UNHEALTHY, DECOMMISSIONING, DECOMMISSIONED, LOST, REBOOTED |
+| state | string | State of the node - valid values are: NEW, RUNNING, UNHEALTHY, DECOMMISSIONING, DECOMMISSIONED, LOST, REBOOTED, SHUTDOWN |
| id | string | The node id |
| nodeHostName | string | The host name of the node |
| nodeHTTPAddress | string | The nodes HTTP address |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/49] hadoop git commit: YARN-7524. Remove unused
FairSchedulerEventLog. (Contributed by Wilfred Spiegelenburg)
Posted by su...@apache.org.
YARN-7524. Remove unused FairSchedulerEventLog. (Contributed by Wilfred Spiegelenburg)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cc9479d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cc9479d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cc9479d
Branch: refs/heads/YARN-5881
Commit: 4cc9479dae2bfb7d14d29b55d103eea9fa35a586
Parents: 738d1a2
Author: Yufei Gu <yu...@apache.org>
Authored: Wed Nov 22 14:18:36 2017 -0800
Committer: Yufei Gu <yu...@apache.org>
Committed: Wed Nov 22 14:18:36 2017 -0800
----------------------------------------------------------------------
.../scheduler/fair/FairScheduler.java | 8 -
.../fair/FairSchedulerConfiguration.java | 16 --
.../scheduler/fair/FairSchedulerEventLog.java | 152 -------------------
.../fair/TestFairSchedulerEventLog.java | 83 ----------
4 files changed, 259 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index b2978d4..661d0a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -177,7 +177,6 @@ public class FairScheduler extends
protected double rackLocalityThreshold; // Cluster threshold for rack locality
protected long nodeLocalityDelayMs; // Delay for node locality
protected long rackLocalityDelayMs; // Delay for rack locality
- private FairSchedulerEventLog eventLog; // Machine-readable event log
protected boolean assignMultiple; // Allocate multiple containers per
// heartbeat
@VisibleForTesting
@@ -404,10 +403,6 @@ public class FairScheduler extends
return continuousSchedulingSleepMs;
}
- public FairSchedulerEventLog getEventLog() {
- return eventLog;
- }
-
/**
* Add a new application to the scheduler, with a given id, queue name, and
* user. This will accept a new app even if the user or queue is above
@@ -875,7 +870,6 @@ public class FairScheduler extends
try {
writeLock.lock();
long start = getClock().getTime();
- eventLog.log("HEARTBEAT", nm.getHostName());
super.nodeUpdate(nm);
FSSchedulerNode fsNode = getFSSchedulerNode(nm.getNodeID());
@@ -1284,8 +1278,6 @@ public class FairScheduler extends
// This stores per-application scheduling information
this.applications = new ConcurrentHashMap<>();
- this.eventLog = new FairSchedulerEventLog();
- eventLog.init(this.conf);
allocConf = new AllocationConfiguration(conf);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 9c9eee6..38e71a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import java.io.File;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -64,12 +63,6 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String ALLOCATION_FILE = CONF_PREFIX + "allocation.file";
protected static final String DEFAULT_ALLOCATION_FILE = "fair-scheduler.xml";
- /** Whether to enable the Fair Scheduler event log */
- public static final String EVENT_LOG_ENABLED = CONF_PREFIX + "event-log-enabled";
- public static final boolean DEFAULT_EVENT_LOG_ENABLED = false;
-
- protected static final String EVENT_LOG_DIR = "eventlog.dir";
-
/** Whether pools can be created that were not specified in the FS configuration file
*/
protected static final String ALLOW_UNDECLARED_POOLS = CONF_PREFIX + "allow-undeclared-pools";
@@ -255,15 +248,6 @@ public class FairSchedulerConfiguration extends Configuration {
return getBoolean(SIZE_BASED_WEIGHT, DEFAULT_SIZE_BASED_WEIGHT);
}
- public boolean isEventLogEnabled() {
- return getBoolean(EVENT_LOG_ENABLED, DEFAULT_EVENT_LOG_ENABLED);
- }
-
- public String getEventlogDir() {
- return get(EVENT_LOG_DIR, new File(System.getProperty("hadoop.log.dir",
- "/tmp/")).getAbsolutePath() + File.separator + "fairscheduler");
- }
-
public long getWaitTimeBeforeNextStarvationCheck() {
return getLong(WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS,
DEFAULT_WAIT_TIME_BEFORE_NEXT_STARVATION_CHECK_MS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
deleted file mode 100644
index 803ce78..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerEventLog.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.log4j.DailyRollingFileAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.PatternLayout;
-import org.apache.log4j.spi.LoggingEvent;
-
-/**
- * Event log used by the fair scheduler for machine-readable debug info.
- * This class uses a log4j rolling file appender to write the log, but uses
- * a custom tab-separated event format of the form:
- * <pre>
- * DATE EVENT_TYPE PARAM_1 PARAM_2 ...
- * </pre>
- * Various event types are used by the fair scheduler. The purpose of logging
- * in this format is to enable tools to parse the history log easily and read
- * internal scheduler variables, rather than trying to make the log human
- * readable. The fair scheduler also logs human readable messages in the
- * JobTracker's main log.
- *
- * Constructing this class creates a disabled log. It must be initialized
- * using {@link FairSchedulerEventLog#init(FairSchedulerConfiguration)} to
- * begin writing to the file.
- */
-@Private
-@Unstable
-class FairSchedulerEventLog {
- private static final Log LOG = LogFactory.getLog(FairSchedulerEventLog.class.getName());
-
- /** Set to true if logging is disabled due to an error. */
- private boolean logDisabled = true;
-
- /**
- * Log directory, set by mapred.fairscheduler.eventlog.location in conf file;
- * defaults to {hadoop.log.dir}/fairscheduler.
- */
- private String logDir;
-
- /**
- * Active log file, which is {LOG_DIR}/hadoop-{user}-fairscheduler.log.
- * Older files are also stored as {LOG_FILE}.date (date format YYYY-MM-DD).
- */
- private String logFile;
-
- /** Log4j appender used to write to the log file */
- private DailyRollingFileAppender appender;
-
- boolean init(FairSchedulerConfiguration conf) {
- if (conf.isEventLogEnabled()) {
- try {
- logDir = conf.getEventlogDir();
- File logDirFile = new File(logDir);
- if (!logDirFile.exists()) {
- if (!logDirFile.mkdirs()) {
- throw new IOException(
- "Mkdirs failed to create " + logDirFile.toString());
- }
- }
- String username = System.getProperty("user.name");
- logFile = String.format("%s%shadoop-%s-fairscheduler.log",
- logDir, File.separator, username);
- logDisabled = false;
- PatternLayout layout = new PatternLayout("%d{ISO8601}\t%m%n");
- appender = new DailyRollingFileAppender(layout, logFile, "'.'yyyy-MM-dd");
- appender.activateOptions();
- LOG.info("Initialized fair scheduler event log, logging to " + logFile);
- } catch (IOException e) {
- LOG.error(
- "Failed to initialize fair scheduler event log. Disabling it.", e);
- logDisabled = true;
- }
- } else {
- logDisabled = true;
- }
- return !(logDisabled);
- }
-
- /**
- * Log an event, writing a line in the log file of the form
- * <pre>
- * DATE EVENT_TYPE PARAM_1 PARAM_2 ...
- * </pre>
- */
- synchronized void log(String eventType, Object... params) {
- try {
- if (logDisabled)
- return;
- StringBuffer buffer = new StringBuffer();
- buffer.append(eventType);
- for (Object param: params) {
- buffer.append("\t");
- buffer.append(param);
- }
- String message = buffer.toString();
- Logger logger = Logger.getLogger(getClass());
- appender.append(new LoggingEvent("", logger, Level.INFO, message, null));
- } catch (Exception e) {
- LOG.error("Failed to append to fair scheduler event log", e);
- logDisabled = true;
- }
- }
-
- /**
- * Flush and close the log.
- */
- synchronized void shutdown() {
- try {
- if (appender != null)
- appender.close();
- } catch (Exception e) {
- LOG.error("Failed to close fair scheduler event log", e);
- logDisabled = true;
- }
- }
-
- synchronized boolean isEnabled() {
- return !logDisabled;
- }
-
- public String getLogFile() {
- return logFile;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cc9479d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
deleted file mode 100644
index 09c2370..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
-import org.junit.Assert;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestFairSchedulerEventLog {
- private File logFile;
- private FairScheduler scheduler;
- private ResourceManager resourceManager;
-
- @Before
- public void setUp() throws IOException {
- scheduler = new FairScheduler();
-
- Configuration conf = new YarnConfiguration();
- conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
- ResourceScheduler.class);
- conf.set(FairSchedulerConfiguration.EVENT_LOG_ENABLED, "true");
-
- // All tests assume only one assignment per node update
- conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
- resourceManager = new MockRM(conf);
- ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
- scheduler.init(conf);
- scheduler.start();
- scheduler.reinitialize(conf, resourceManager.getRMContext());
- }
-
- /**
- * Make sure the scheduler creates the event log.
- */
- @Test
- public void testCreateEventLog() throws IOException {
- FairSchedulerEventLog eventLog = scheduler.getEventLog();
-
- logFile = new File(eventLog.getLogFile());
- Assert.assertTrue(logFile.exists());
- }
-
- @After
- public void tearDown() {
- logFile.delete();
- logFile.getParentFile().delete(); // fairscheduler/
- if (scheduler != null) {
- scheduler.stop();
- scheduler = null;
- }
- if (resourceManager != null) {
- resourceManager.stop();
- resourceManager = null;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
index f846689..96de8e4 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ABlockOutputStream.java
@@ -22,17 +22,16 @@ import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.Callable;
+import java.util.Locale;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import com.amazonaws.AmazonClientException;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressEventType;
import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
@@ -47,8 +46,9 @@ import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryPolicy;
+import org.apache.hadoop.fs.StreamCapabilities;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.PutTracker;
import org.apache.hadoop.util.Progressable;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
@@ -65,7 +65,8 @@ import static org.apache.hadoop.fs.s3a.Statistic.*;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-class S3ABlockOutputStream extends OutputStream {
+class S3ABlockOutputStream extends OutputStream implements
+ StreamCapabilities {
private static final Logger LOG =
LoggerFactory.getLogger(S3ABlockOutputStream.class);
@@ -87,14 +88,6 @@ class S3ABlockOutputStream extends OutputStream {
private final ListeningExecutorService executorService;
/**
- * Retry policy for multipart commits; not all AWS SDK versions retry that.
- */
- private final RetryPolicy retryPolicy =
- RetryPolicies.retryUpToMaximumCountWithProportionalSleep(
- 5,
- 2000,
- TimeUnit.MILLISECONDS);
- /**
* Factory for blocks.
*/
private final S3ADataBlocks.BlockFactory blockFactory;
@@ -120,7 +113,12 @@ class S3ABlockOutputStream extends OutputStream {
/**
* Write operation helper; encapsulation of the filesystem operations.
*/
- private final S3AFileSystem.WriteOperationHelper writeOperationHelper;
+ private final WriteOperationHelper writeOperationHelper;
+
+ /**
+ * Track multipart put operation.
+ */
+ private final PutTracker putTracker;
/**
* An S3A output stream which uploads partitions in a separate pool of
@@ -138,6 +136,7 @@ class S3ABlockOutputStream extends OutputStream {
* @param blockFactory factory for creating stream destinations
* @param statistics stats for this stream
* @param writeOperationHelper state of the write operation.
+ * @param putTracker put tracking for commit support
* @throws IOException on any problem
*/
S3ABlockOutputStream(S3AFileSystem fs,
@@ -147,7 +146,8 @@ class S3ABlockOutputStream extends OutputStream {
long blockSize,
S3ADataBlocks.BlockFactory blockFactory,
S3AInstrumentation.OutputStreamStatistics statistics,
- S3AFileSystem.WriteOperationHelper writeOperationHelper)
+ WriteOperationHelper writeOperationHelper,
+ PutTracker putTracker)
throws IOException {
this.fs = fs;
this.key = key;
@@ -155,6 +155,7 @@ class S3ABlockOutputStream extends OutputStream {
this.blockSize = (int) blockSize;
this.statistics = statistics;
this.writeOperationHelper = writeOperationHelper;
+ this.putTracker = putTracker;
Preconditions.checkArgument(blockSize >= Constants.MULTIPART_MIN_SIZE,
"Block size is too small: %d", blockSize);
this.executorService = MoreExecutors.listeningDecorator(executorService);
@@ -166,7 +167,11 @@ class S3ABlockOutputStream extends OutputStream {
// writes a 0-byte entry.
createBlockIfNeeded();
LOG.debug("Initialized S3ABlockOutputStream for {}" +
- " output to {}", writeOperationHelper, activeBlock);
+ " output to {}", key, activeBlock);
+ if (putTracker.initialize()) {
+ LOG.debug("Put tracker requests multipart upload");
+ initMultipartUpload();
+ }
}
/**
@@ -299,10 +304,7 @@ class S3ABlockOutputStream extends OutputStream {
private synchronized void uploadCurrentBlock() throws IOException {
Preconditions.checkState(hasActiveBlock(), "No active block");
LOG.debug("Writing block # {}", blockCount);
- if (multiPartUpload == null) {
- LOG.debug("Initiating Multipart upload");
- multiPartUpload = new MultiPartUpload();
- }
+ initMultipartUpload();
try {
multiPartUpload.uploadBlockAsync(getActiveBlock());
bytesSubmitted += getActiveBlock().dataSize();
@@ -313,6 +315,20 @@ class S3ABlockOutputStream extends OutputStream {
}
/**
+ * Init multipart upload. Assumption: this is called from
+ * a synchronized block.
+ * Note that this makes a blocking HTTPS request to the far end, so
+ * can take time and potentially fail.
+ * @throws IOException failure to initialize the upload
+ */
+ private void initMultipartUpload() throws IOException {
+ if (multiPartUpload == null) {
+ LOG.debug("Initiating Multipart upload");
+ multiPartUpload = new MultiPartUpload(key);
+ }
+ }
+
+ /**
* Close the stream.
*
* This will not return until the upload is complete
@@ -342,22 +358,35 @@ class S3ABlockOutputStream extends OutputStream {
// This must happen even if there is no data, so that 0 byte files
// are created.
bytes = putObject();
+ bytesSubmitted = bytes;
}
} else {
- // there has already been at least one block scheduled for upload;
- // put up the current then wait
- if (hasBlock && block.hasData()) {
+ // there's an MPU in progress';
+ // IF there is more data to upload, or no data has yet been uploaded,
+ // PUT the final block
+ if (hasBlock &&
+ (block.hasData() || multiPartUpload.getPartsSubmitted() == 0)) {
//send last part
uploadCurrentBlock();
}
// wait for the partial uploads to finish
final List<PartETag> partETags =
multiPartUpload.waitForAllPartUploads();
- // then complete the operation
- multiPartUpload.complete(partETags);
bytes = bytesSubmitted;
+ // then complete the operation
+ if (putTracker.aboutToComplete(multiPartUpload.getUploadId(),
+ partETags,
+ bytes)) {
+ multiPartUpload.complete(partETags);
+ } else {
+ LOG.info("File {} will be visible when the job is committed", key);
+ }
+ }
+ if (!putTracker.outputImmediatelyVisible()) {
+ // track the number of bytes uploaded as commit operations.
+ statistics.commitUploaded(bytes);
}
- LOG.debug("Upload complete for {}", writeOperationHelper);
+ LOG.debug("Upload complete to {} by {}", key, writeOperationHelper);
} catch (IOException ioe) {
writeOperationHelper.writeFailed(ioe);
throw ioe;
@@ -367,7 +396,7 @@ class S3ABlockOutputStream extends OutputStream {
closeAll(LOG, statistics);
clearActiveBlock();
}
- // All end of write operations, including deleting fake parent directories
+ // Note end of write. This does not change the state of the remote FS.
writeOperationHelper.writeSuccessful(bytes);
}
@@ -387,8 +416,9 @@ class S3ABlockOutputStream extends OutputStream {
int size = block.dataSize();
final S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
final PutObjectRequest putObjectRequest = uploadData.hasFile() ?
- writeOperationHelper.newPutRequest(uploadData.getFile()) :
- writeOperationHelper.newPutRequest(uploadData.getUploadStream(), size);
+ writeOperationHelper.createPutObjectRequest(key, uploadData.getFile())
+ : writeOperationHelper.createPutObjectRequest(key,
+ uploadData.getUploadStream(), size);
long transferQueueTime = now();
BlockUploadProgress callback =
new BlockUploadProgress(
@@ -396,18 +426,13 @@ class S3ABlockOutputStream extends OutputStream {
putObjectRequest.setGeneralProgressListener(callback);
statistics.blockUploadQueued(size);
ListenableFuture<PutObjectResult> putObjectResult =
- executorService.submit(new Callable<PutObjectResult>() {
- @Override
- public PutObjectResult call() throws Exception {
- PutObjectResult result;
- try {
- // the putObject call automatically closes the input
- // stream afterwards.
- result = writeOperationHelper.putObject(putObjectRequest);
- } finally {
- closeAll(LOG, uploadData, block);
- }
- return result;
+ executorService.submit(() -> {
+ try {
+ // the putObject call automatically closes the input
+ // stream afterwards.
+ return writeOperationHelper.putObject(putObjectRequest);
+ } finally {
+ closeAll(LOG, uploadData, block);
}
});
clearActiveBlock();
@@ -460,20 +485,82 @@ class S3ABlockOutputStream extends OutputStream {
}
/**
+ * Return the stream capabilities.
+ * This stream always returns false when queried about hflush and hsync.
+ * If asked about {@link CommitConstants#STREAM_CAPABILITY_MAGIC_OUTPUT}
+ * it will return true iff this is an active "magic" output stream.
+ * @param capability string to query the stream support for.
+ * @return true if the capability is supported by this instance.
+ */
+ @Override
+ public boolean hasCapability(String capability) {
+ switch (capability.toLowerCase(Locale.ENGLISH)) {
+
+ // does the output stream have delayed visibility
+ case CommitConstants.STREAM_CAPABILITY_MAGIC_OUTPUT:
+ return !putTracker.outputImmediatelyVisible();
+
+ // The flush/sync options are absolutely not supported
+ case "hflush":
+ case "hsync":
+ return false;
+
+ default:
+ return false;
+ }
+ }
+
+ /**
* Multiple partition upload.
*/
private class MultiPartUpload {
private final String uploadId;
private final List<ListenableFuture<PartETag>> partETagsFutures;
+ private int partsSubmitted;
+ private int partsUploaded;
+ private long bytesSubmitted;
- MultiPartUpload() throws IOException {
- this.uploadId = writeOperationHelper.initiateMultiPartUpload();
+ MultiPartUpload(String key) throws IOException {
+ this.uploadId = writeOperationHelper.initiateMultiPartUpload(key);
this.partETagsFutures = new ArrayList<>(2);
LOG.debug("Initiated multi-part upload for {} with " +
"id '{}'", writeOperationHelper, uploadId);
}
/**
+ * Get a count of parts submitted.
+ * @return the number of parts submitted; will always be >= the
+ * value of {@link #getPartsUploaded()}
+ */
+ public int getPartsSubmitted() {
+ return partsSubmitted;
+ }
+
+ /**
+ * Count of parts actually uploaded.
+ * @return the count of successfully completed part uploads.
+ */
+ public int getPartsUploaded() {
+ return partsUploaded;
+ }
+
+ /**
+ * Get the upload ID; will be null after construction completes.
+ * @return the upload ID
+ */
+ public String getUploadId() {
+ return uploadId;
+ }
+
+ /**
+ * Get the count of bytes submitted.
+ * @return the current upload size.
+ */
+ public long getBytesSubmitted() {
+ return bytesSubmitted;
+ }
+
+ /**
* Upload a block of data.
* This will take the block
* @param block block to upload
@@ -481,17 +568,22 @@ class S3ABlockOutputStream extends OutputStream {
*/
private void uploadBlockAsync(final S3ADataBlocks.DataBlock block)
throws IOException {
- LOG.debug("Queueing upload of {}", block);
+ LOG.debug("Queueing upload of {} for upload {}", block, uploadId);
+ Preconditions.checkNotNull(uploadId, "Null uploadId");
+ partsSubmitted++;
final int size = block.dataSize();
+ bytesSubmitted += size;
final S3ADataBlocks.BlockUploadData uploadData = block.startUpload();
final int currentPartNumber = partETagsFutures.size() + 1;
final UploadPartRequest request =
writeOperationHelper.newUploadPartRequest(
+ key,
uploadId,
currentPartNumber,
size,
uploadData.getUploadStream(),
- uploadData.getFile());
+ uploadData.getFile(),
+ 0L);
long transferQueueTime = now();
BlockUploadProgress callback =
@@ -500,24 +592,22 @@ class S3ABlockOutputStream extends OutputStream {
request.setGeneralProgressListener(callback);
statistics.blockUploadQueued(block.dataSize());
ListenableFuture<PartETag> partETagFuture =
- executorService.submit(new Callable<PartETag>() {
- @Override
- public PartETag call() throws Exception {
- // this is the queued upload operation
- LOG.debug("Uploading part {} for id '{}'", currentPartNumber,
- uploadId);
- // do the upload
- PartETag partETag;
- try {
- partETag = fs.uploadPart(request).getPartETag();
- LOG.debug("Completed upload of {} to part {}", block,
- partETag.getETag());
- LOG.debug("Stream statistics of {}", statistics);
- } finally {
- // close the stream and block
- closeAll(LOG, uploadData, block);
- }
+ executorService.submit(() -> {
+ // this is the queued upload operation
+ // do the upload
+ try {
+ LOG.debug("Uploading part {} for id '{}'",
+ currentPartNumber, uploadId);
+ PartETag partETag = writeOperationHelper.uploadPart(request)
+ .getPartETag();
+ LOG.debug("Completed upload of {} to part {}",
+ block, partETag.getETag());
+ LOG.debug("Stream statistics of {}", statistics);
+ partsUploaded++;
return partETag;
+ } finally {
+ // close the stream and block
+ closeAll(LOG, uploadData, block);
}
});
partETagsFutures.add(partETagFuture);
@@ -558,28 +648,18 @@ class S3ABlockOutputStream extends OutputStream {
* @param partETags list of partial uploads
* @throws IOException on any problem
*/
- private CompleteMultipartUploadResult complete(List<PartETag> partETags)
+ private void complete(List<PartETag> partETags)
throws IOException {
- int retryCount = 0;
- AmazonClientException lastException;
- String operation =
- String.format("Completing multi-part upload for key '%s'," +
- " id '%s' with %s partitions ",
- key, uploadId, partETags.size());
- do {
- try {
- LOG.debug(operation);
- return writeOperationHelper.completeMultipartUpload(
- uploadId,
- partETags);
- } catch (AmazonClientException e) {
- lastException = e;
- statistics.exceptionInMultipartComplete();
- }
- } while (shouldRetry(operation, lastException, retryCount++));
- // this point is only reached if the operation failed more than
- // the allowed retry count
- throw translateException(operation, key, lastException);
+ AtomicInteger errorCount = new AtomicInteger(0);
+ try {
+ writeOperationHelper.completeMPUwithRetries(key,
+ uploadId,
+ partETags,
+ bytesSubmitted,
+ errorCount);
+ } finally {
+ statistics.exceptionInMultipartComplete(errorCount.get());
+ }
}
/**
@@ -590,57 +670,14 @@ class S3ABlockOutputStream extends OutputStream {
int retryCount = 0;
AmazonClientException lastException;
fs.incrementStatistic(OBJECT_MULTIPART_UPLOAD_ABORTED);
- String operation =
- String.format("Aborting multi-part upload for '%s', id '%s",
- writeOperationHelper, uploadId);
- do {
- try {
- LOG.debug(operation);
- writeOperationHelper.abortMultipartUpload(uploadId);
- return;
- } catch (AmazonClientException e) {
- lastException = e;
- statistics.exceptionInMultipartAbort();
- }
- } while (shouldRetry(operation, lastException, retryCount++));
- // this point is only reached if the operation failed more than
- // the allowed retry count
- LOG.warn("Unable to abort multipart upload, you may need to purge " +
- "uploaded parts", lastException);
- }
-
- /**
- * Predicate to determine whether a failed operation should
- * be attempted again.
- * If a retry is advised, the exception is automatically logged and
- * the filesystem statistic {@link Statistic#IGNORED_ERRORS} incremented.
- * The method then sleeps for the sleep time suggested by the sleep policy;
- * if the sleep is interrupted then {@code Thread.interrupted()} is set
- * to indicate the thread was interrupted; then false is returned.
- *
- * @param operation operation for log message
- * @param e exception raised.
- * @param retryCount number of retries already attempted
- * @return true if another attempt should be made
- */
- private boolean shouldRetry(String operation,
- AmazonClientException e,
- int retryCount) {
try {
- RetryPolicy.RetryAction retryAction =
- retryPolicy.shouldRetry(e, retryCount, 0, true);
- boolean retry = retryAction == RetryPolicy.RetryAction.RETRY;
- if (retry) {
- fs.incrementStatistic(IGNORED_ERRORS);
- LOG.info("Retrying {} after exception ", operation, e);
- Thread.sleep(retryAction.delayMillis);
- }
- return retry;
- } catch (InterruptedException ex) {
- Thread.currentThread().interrupt();
- return false;
- } catch (Exception ignored) {
- return false;
+ writeOperationHelper.abortMultipartUpload(key, uploadId,
+ (text, e, r, i) -> statistics.exceptionInMultipartAbort());
+ } catch (IOException e) {
+ // this point is only reached if the operation failed more than
+ // the allowed retry count
+ LOG.warn("Unable to abort multipart upload,"
+ + " you may need to purge uploaded parts", e);
}
}
@@ -718,7 +755,7 @@ class S3ABlockOutputStream extends OutputStream {
private static class ProgressableListener implements ProgressListener {
private final Progressable progress;
- public ProgressableListener(Progressable progress) {
+ ProgressableListener(Progressable progress) {
this.progress = progress;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
index 9bc8dcd..0e3bca5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ADataBlocks.java
@@ -401,7 +401,7 @@ final class S3ADataBlocks {
}
/**
- * InputStream backed by the internal byte array
+ * InputStream backed by the internal byte array.
*
* @return
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 2171957..b08a134 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -24,6 +24,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.net.URI;
+import java.nio.file.AccessDeniedException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -31,6 +34,7 @@ import java.util.Date;
import java.util.EnumSet;
import java.util.HashSet;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.Objects;
@@ -44,19 +48,18 @@ import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
-import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
-import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsV2Request;
import com.amazonaws.services.s3.model.MultiObjectDeleteException;
+import com.amazonaws.services.s3.model.MultipartUpload;
import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3ObjectSummary;
@@ -68,6 +71,7 @@ import com.amazonaws.services.s3.transfer.Copy;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerConfiguration;
import com.amazonaws.services.s3.transfer.Upload;
+import com.amazonaws.services.s3.transfer.model.UploadResult;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.event.ProgressEvent;
import com.google.common.annotations.VisibleForTesting;
@@ -94,20 +98,24 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.StreamCapabilities;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.PutTracker;
+import org.apache.hadoop.fs.s3a.commit.MagicCommitIntegration;
import org.apache.hadoop.fs.s3a.s3guard.DirListingMetadata;
import org.apache.hadoop.fs.s3a.s3guard.MetadataStoreListFilesIterator;
import org.apache.hadoop.fs.s3a.s3guard.MetadataStore;
import org.apache.hadoop.fs.s3a.s3guard.PathMetadata;
import org.apache.hadoop.fs.s3a.s3guard.S3Guard;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
+import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ReflectionUtils;
import static org.apache.hadoop.fs.s3a.Constants.*;
-import static org.apache.hadoop.fs.s3a.Listing.ACCEPT_ALL;
+import static org.apache.hadoop.fs.s3a.Invoker.*;
import static org.apache.hadoop.fs.s3a.S3AUtils.*;
import static org.apache.hadoop.fs.s3a.Statistic.*;
@@ -129,15 +137,31 @@ import org.slf4j.LoggerFactory;
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class S3AFileSystem extends FileSystem {
+public class S3AFileSystem extends FileSystem implements StreamCapabilities {
/**
* Default blocksize as used in blocksize and FS status queries.
*/
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
+
+ /**
+ * This declared delete as idempotent.
+ * This is an "interesting" topic in past Hadoop FS work.
+ * Essentially: with a single caller, DELETE is idempotent
+ * but in a shared filesystem, it is is very much not so.
+ * Here, on the basis that isn't a filesystem with consistency guarantees,
+ * retryable results in files being deleted.
+ */
+ public static final boolean DELETE_CONSIDERED_IDEMPOTENT = true;
private URI uri;
private Path workingDir;
private String username;
private AmazonS3 s3;
+ // initial callback policy is fail-once; it's there just to assist
+ // some mock tests and other codepaths trying to call the low level
+ // APIs on an uninitialized filesystem.
+ private Invoker invoker = new Invoker(RetryPolicies.TRY_ONCE_THEN_FAIL,
+ Invoker.LOG_EVENT);
+ private final Retried onRetry = this::operationRetried;
private String bucket;
private int maxKeys;
private Listing listing;
@@ -154,7 +178,8 @@ public class S3AFileSystem extends FileSystem {
private CannedAccessControlList cannedACL;
private S3AEncryptionMethods serverSideEncryptionAlgorithm;
private S3AInstrumentation instrumentation;
- private S3AStorageStatistics storageStatistics;
+ private final S3AStorageStatistics storageStatistics =
+ createStorageStatistics();
private long readAhead;
private S3AInputPolicy inputPolicy;
private final AtomicBoolean closed = new AtomicBoolean(false);
@@ -167,6 +192,7 @@ public class S3AFileSystem extends FileSystem {
private S3ADataBlocks.BlockFactory blockFactory;
private int blockOutputActiveBlocks;
private boolean useListV1;
+ private MagicCommitIntegration committerIntegration;
/** Add any deprecated keys. */
@SuppressWarnings("deprecation")
@@ -194,9 +220,10 @@ public class S3AFileSystem extends FileSystem {
*/
public void initialize(URI name, Configuration originalConf)
throws IOException {
- uri = S3xLoginHelper.buildFSURI(name);
+ setUri(name);
// get the host; this is guaranteed to be non-null, non-empty
bucket = name.getHost();
+ LOG.debug("Initializing S3AFileSystem for {}", bucket);
// clone the configuration into one with propagated bucket options
Configuration conf = propagateBucketOptions(originalConf, bucket);
patchSecurityCredentialProviders(conf);
@@ -216,6 +243,7 @@ public class S3AFileSystem extends FileSystem {
S3ClientFactory.class);
s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
.createS3Client(name);
+ invoker = new Invoker(new S3ARetryPolicy(getConf()), onRetry);
maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1);
listing = new Listing(this);
@@ -230,15 +258,6 @@ public class S3AFileSystem extends FileSystem {
readAhead = longBytesOption(conf, READAHEAD_RANGE,
DEFAULT_READAHEAD_RANGE, 0);
- storageStatistics = (S3AStorageStatistics)
- GlobalStorageStatistics.INSTANCE
- .put(S3AStorageStatistics.NAME,
- new GlobalStorageStatistics.StorageStatisticsProvider() {
- @Override
- public StorageStatistics provide() {
- return new S3AStorageStatistics();
- }
- });
int maxThreads = conf.getInt(MAX_THREADS, DEFAULT_MAX_THREADS);
if (maxThreads < 2) {
@@ -274,11 +293,17 @@ public class S3AFileSystem extends FileSystem {
verifyBucketExists();
- initMultipartUploads(conf);
-
serverSideEncryptionAlgorithm = getEncryptionAlgorithm(conf);
inputPolicy = S3AInputPolicy.getPolicy(
conf.getTrimmed(INPUT_FADVISE, INPUT_FADV_NORMAL));
+ LOG.debug("Input fadvise policy = {}", inputPolicy);
+ boolean magicCommitterEnabled = conf.getBoolean(
+ CommitConstants.MAGIC_COMMITTER_ENABLED,
+ CommitConstants.DEFAULT_MAGIC_COMMITTER_ENABLED);
+ LOG.debug("Filesystem support for magic committers {} enabled",
+ magicCommitterEnabled ? "is" : "is not");
+ committerIntegration = new MagicCommitIntegration(
+ this, magicCommitterEnabled);
boolean blockUploadEnabled = conf.getBoolean(FAST_UPLOAD, true);
@@ -295,13 +320,14 @@ public class S3AFileSystem extends FileSystem {
" queue limit={}",
blockOutputBuffer, partSize, blockOutputActiveBlocks);
- metadataStore = S3Guard.getMetadataStore(this);
+ setMetadataStore(S3Guard.getMetadataStore(this));
allowAuthoritative = conf.getBoolean(METADATASTORE_AUTHORITATIVE,
DEFAULT_METADATASTORE_AUTHORITATIVE);
if (hasMetadataStore()) {
LOG.debug("Using metadata store {}, authoritative={}",
getMetadataStore(), allowAuthoritative);
}
+ initMultipartUploads(conf);
} catch (AmazonClientException e) {
throw translateException("initializing ", new Path(name), e);
}
@@ -309,27 +335,29 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Create the storage statistics or bind to an existing one.
+ * @return a storage statistics instance.
+ */
+ protected static S3AStorageStatistics createStorageStatistics() {
+ return (S3AStorageStatistics)
+ GlobalStorageStatistics.INSTANCE
+ .put(S3AStorageStatistics.NAME,
+ () -> new S3AStorageStatistics());
+ }
+
+ /**
* Verify that the bucket exists. This does not check permissions,
* not even read access.
+ * Retry policy: retrying, translated.
* @throws FileNotFoundException the bucket is absent
* @throws IOException any other problem talking to S3
*/
+ @Retries.RetryTranslated
protected void verifyBucketExists()
throws FileNotFoundException, IOException {
- try {
- if (!s3.doesBucketExist(bucket)) {
- throw new FileNotFoundException("Bucket " + bucket + " does not exist");
- }
- } catch (AmazonS3Exception e) {
- // this is a sign of a serious startup problem so do dump everything
- LOG.warn(stringify(e), e);
- throw translateException("doesBucketExist", bucket, e);
- } catch (AmazonServiceException e) {
- // this is a sign of a serious startup problem so do dump everything
- LOG.warn(stringify(e), e);
- throw translateException("doesBucketExist", bucket, e);
- } catch (AmazonClientException e) {
- throw translateException("doesBucketExist", bucket, e);
+ if (!invoker.retry("doesBucketExist", bucket, true,
+ () -> s3.doesBucketExist(bucket))) {
+ throw new FileNotFoundException("Bucket " + bucket + " does not exist");
}
}
@@ -362,6 +390,7 @@ public class S3AFileSystem extends FileSystem {
}
}
+ @Retries.RetryTranslated
private void initMultipartUploads(Configuration conf) throws IOException {
boolean purgeExistingMultipart = conf.getBoolean(PURGE_EXISTING_MULTIPART,
DEFAULT_PURGE_EXISTING_MULTIPART);
@@ -369,24 +398,34 @@ public class S3AFileSystem extends FileSystem {
PURGE_EXISTING_MULTIPART_AGE, DEFAULT_PURGE_EXISTING_MULTIPART_AGE, 0);
if (purgeExistingMultipart) {
- Date purgeBefore =
- new Date(new Date().getTime() - purgeExistingMultipartAge * 1000);
-
try {
- transfers.abortMultipartUploads(bucket, purgeBefore);
- } catch (AmazonServiceException e) {
- if (e.getStatusCode() == 403) {
- instrumentation.errorIgnored();
- LOG.debug("Failed to purging multipart uploads against {}," +
- " FS may be read only", bucket, e);
- } else {
- throw translateException("purging multipart uploads", bucket, e);
- }
+ abortOutstandingMultipartUploads(purgeExistingMultipartAge);
+ } catch (AccessDeniedException e) {
+ instrumentation.errorIgnored();
+ LOG.debug("Failed to purge multipart uploads against {}," +
+ " FS may be read only", bucket);
}
}
}
/**
+ * Abort all outstanding MPUs older than a given age.
+ * @param seconds time in seconds
+ * @throws IOException on any failure, other than 403 "permission denied"
+ */
+ @Retries.RetryTranslated
+ public void abortOutstandingMultipartUploads(long seconds)
+ throws IOException {
+ Preconditions.checkArgument(seconds >= 0);
+ Date purgeBefore =
+ new Date(new Date().getTime() - seconds * 1000);
+ LOG.debug("Purging outstanding multipart uploads older than {}",
+ purgeBefore);
+ invoker.retry("Purging multipart uploads", bucket, true,
+ () -> transfers.abortMultipartUploads(bucket, purgeBefore));
+ }
+
+ /**
* Return the protocol scheme for the FileSystem.
*
* @return "s3a"
@@ -404,6 +443,16 @@ public class S3AFileSystem extends FileSystem {
return uri;
}
+ /**
+ * Set the URI field through {@link S3xLoginHelper}.
+ * Exported for testing.
+ * @param uri filesystem URI.
+ */
+ @VisibleForTesting
+ protected void setUri(URI uri) {
+ this.uri = S3xLoginHelper.buildFSURI(uri);
+ }
+
@Override
public int getDefaultPort() {
return Constants.S3A_DEFAULT_PORT;
@@ -411,6 +460,7 @@ public class S3AFileSystem extends FileSystem {
/**
* Returns the S3 client used by this filesystem.
+ * This is for internal use within the S3A code itself.
* @return AmazonS3Client
*/
AmazonS3 getAmazonS3Client() {
@@ -418,34 +468,55 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Returns the S3 client used by this filesystem.
+ * <i>Warning: this must only be used for testing, as it bypasses core
+ * S3A operations. </i>
+ * @param reason a justification for requesting access.
+ * @return AmazonS3Client
+ */
+ @VisibleForTesting
+ public AmazonS3 getAmazonS3ClientForTesting(String reason) {
+ LOG.warn("Access to S3A client requested, reason {}", reason);
+ return s3;
+ }
+
+ /**
+ * Set the client -used in mocking tests to force in a different client.
+ * @param client client.
+ */
+ protected void setAmazonS3Client(AmazonS3 client) {
+ Preconditions.checkNotNull(client, "client");
+ LOG.debug("Setting S3 client to {}", client);
+ s3 = client;
+ }
+
+ /**
* Get the region of a bucket.
* @return the region in which a bucket is located
* @throws IOException on any failure.
*/
+ @Retries.RetryTranslated
public String getBucketLocation() throws IOException {
return getBucketLocation(bucket);
}
/**
* Get the region of a bucket.
+ * Retry policy: retrying, translated.
* @param bucketName the name of the bucket
* @return the region in which a bucket is located
* @throws IOException on any failure.
*/
+ @Retries.RetryTranslated
public String getBucketLocation(String bucketName) throws IOException {
- try {
- return s3.getBucketLocation(bucketName);
- } catch (AmazonClientException e) {
- throw translateException("getBucketLocation()",
- bucketName, e);
- }
+ return invoker.retry("getBucketLocation()", bucketName, true,
+ ()-> s3.getBucketLocation(bucketName));
}
/**
- * Returns the read ahead range value used by this filesystem
- * @return
+ * Returns the read ahead range value used by this filesystem.
+ * @return the readahead range
*/
-
@VisibleForTesting
long getReadAheadRange() {
return readAhead;
@@ -473,7 +544,7 @@ public class S3AFileSystem extends FileSystem {
Configuration conf) throws IOException {
if (directoryAllocator == null) {
String bufferDir = conf.get(BUFFER_DIR) != null
- ? BUFFER_DIR : "hadoop.tmp.dir";
+ ? BUFFER_DIR : HADOOP_TMP_DIR;
directoryAllocator = new LocalDirAllocator(bufferDir);
}
return directoryAllocator.createTmpFileForWrite(pathStr, size, conf);
@@ -488,6 +559,23 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Set the bucket.
+ * @param bucket the bucket
+ */
+ @VisibleForTesting
+ protected void setBucket(String bucket) {
+ this.bucket = bucket;
+ }
+
+ /**
+ * Get the canned ACL of this FS.
+ * @return an ACL, if any
+ */
+ CannedAccessControlList getCannedACL() {
+ return cannedACL;
+ }
+
+ /**
* Change the input policy for this FS.
* @param inputPolicy new policy
*/
@@ -538,7 +626,7 @@ public class S3AFileSystem extends FileSystem {
* @param key input key
* @return the path from this key
*/
- private Path keyToPath(String key) {
+ Path keyToPath(String key) {
return new Path("/" + key);
}
@@ -547,7 +635,7 @@ public class S3AFileSystem extends FileSystem {
* @param key input key
* @return the fully qualified path including URI scheme and bucket name.
*/
- Path keyToQualifiedPath(String key) {
+ public Path keyToQualifiedPath(String key) {
return qualify(keyToPath(key));
}
@@ -585,7 +673,7 @@ public class S3AFileSystem extends FileSystem {
public FSDataInputStream open(Path f, int bufferSize)
throws IOException {
- LOG.debug("Opening '{}' for reading.", f);
+ LOG.debug("Opening '{}' for reading; input policy = {}", f, inputPolicy);
final FileStatus fileStatus = getFileStatus(f);
if (fileStatus.isDirectory()) {
throw new FileNotFoundException("Can't open " + f
@@ -603,12 +691,15 @@ public class S3AFileSystem extends FileSystem {
statistics,
instrumentation,
readAhead,
- inputPolicy));
+ inputPolicy,
+ invoker));
}
/**
* Create an FSDataOutputStream at the indicated Path with write-progress
* reporting.
+ * Retry policy: retrying, translated on the getFileStatus() probe.
+ * No data is uploaded to S3 in this call, so retry issues related to that.
* @param f the file name to open
* @param permission the permission to set.
* @param overwrite if a file with this name already exists, then if true,
@@ -625,42 +716,60 @@ public class S3AFileSystem extends FileSystem {
public FSDataOutputStream create(Path f, FsPermission permission,
boolean overwrite, int bufferSize, short replication, long blockSize,
Progressable progress) throws IOException {
- String key = pathToKey(f);
+ final Path path = qualify(f);
+ String key = pathToKey(path);
FileStatus status = null;
try {
// get the status or throw an FNFE
- status = getFileStatus(f);
+ status = getFileStatus(path);
// if the thread reaches here, there is something at the path
if (status.isDirectory()) {
// path references a directory: automatic error
- throw new FileAlreadyExistsException(f + " is a directory");
+ throw new FileAlreadyExistsException(path + " is a directory");
}
if (!overwrite) {
// path references a file and overwrite is disabled
- throw new FileAlreadyExistsException(f + " already exists");
+ throw new FileAlreadyExistsException(path + " already exists");
}
- LOG.debug("Overwriting file {}", f);
+ LOG.debug("Overwriting file {}", path);
} catch (FileNotFoundException e) {
// this means the file is not found
}
instrumentation.fileCreated();
+ PutTracker putTracker =
+ committerIntegration.createTracker(path, key);
+ String destKey = putTracker.getDestKey();
return new FSDataOutputStream(
new S3ABlockOutputStream(this,
- key,
+ destKey,
new SemaphoredDelegatingExecutor(boundedThreadPool,
blockOutputActiveBlocks, true),
progress,
partSize,
blockFactory,
instrumentation.newOutputStreamStatistics(statistics),
- new WriteOperationHelper(key)
- ),
+ createWriteOperationHelper(),
+ putTracker),
null);
}
/**
+ * Create a new {@code WriteOperationHelper} instance.
+ *
+ * This class permits other low-level operations against the store.
+ * It is unstable and
+ * only intended for code with intimate knowledge of the object store.
+ * If using this, be prepared for changes even on minor point releases.
+ * @return a new helper.
+ */
+ @InterfaceAudience.Private
+ public WriteOperationHelper createWriteOperationHelper() {
+ return new WriteOperationHelper(this);
+ }
+
+ /**
* {@inheritDoc}
* @throws FileNotFoundException if the parent directory is not present -or
* is not a directory.
@@ -883,7 +992,7 @@ public class S3AFileSystem extends FileSystem {
keysToDelete.add(new DeleteObjectsRequest.KeyVersion(dstKey));
}
- Path parentPath = keyToPath(srcKey);
+ Path parentPath = keyToQualifiedPath(srcKey);
RemoteIterator<LocatedFileStatus> iterator = listFilesAndEmptyDirectories(
parentPath, true);
while (iterator.hasNext()) {
@@ -940,7 +1049,7 @@ public class S3AFileSystem extends FileSystem {
if (src.getParent() != dst.getParent()) {
deleteUnnecessaryFakeDirectories(dst.getParent());
- createFakeDirectoryIfNecessary(src.getParent());
+ maybeCreateFakeParentDirectory(src);
}
return true;
}
@@ -978,6 +1087,7 @@ public class S3AFileSystem extends FileSystem {
/** For testing only. See ITestS3GuardEmptyDirs. */
@VisibleForTesting
void setMetadataStore(MetadataStore ms) {
+ Preconditions.checkNotNull(ms);
metadataStore = ms;
}
@@ -1018,6 +1128,48 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Callback when an operation was retried.
+ * Increments the statistics of ignored errors or throttled requests,
+ * depending up on the exception class.
+ * @param ex exception.
+ */
+ public void operationRetried(Exception ex) {
+ Statistic stat = isThrottleException(ex)
+ ? STORE_IO_THROTTLED
+ : IGNORED_ERRORS;
+ instrumentation.incrementCounter(stat, 1);
+ storageStatistics.incrementCounter(stat, 1);
+ }
+
+ /**
+ * Callback from {@link Invoker} when an operation is retried.
+ * @param text text of the operation
+ * @param ex exception
+ * @param retries number of retries
+ * @param idempotent is the method idempotent
+ */
+ public void operationRetried(
+ String text,
+ Exception ex,
+ int retries,
+ boolean idempotent) {
+ operationRetried(ex);
+ }
+
+ /**
+ * Callback from {@link Invoker} when an operation against a metastore
+ * is retried.
+ * @param ex exception
+ * @param retries number of retries
+ * @param idempotent is the method idempotent
+ */
+ public void metastoreOperationRetried(Exception ex,
+ int retries,
+ boolean idempotent) {
+ operationRetried(ex);
+ }
+
+ /**
* Get the storage statistics of this filesystem.
* @return the storage statistics
*/
@@ -1028,11 +1180,13 @@ public class S3AFileSystem extends FileSystem {
/**
* Request object metadata; increments counters in the process.
+ * Retry policy: retry untranslated.
* @param key key
* @return the metadata
+ * @throws IOException if the retry invocation raises one (it shouldn't).
*/
- protected ObjectMetadata getObjectMetadata(String key) {
- incrementStatistic(OBJECT_METADATA_REQUESTS);
+ @Retries.RetryRaw
+ protected ObjectMetadata getObjectMetadata(String key) throws IOException {
GetObjectMetadataRequest request =
new GetObjectMetadataRequest(bucket, key);
//SSE-C requires to be filled in if enabled for object metadata
@@ -1040,7 +1194,11 @@ public class S3AFileSystem extends FileSystem {
StringUtils.isNotBlank(getServerSideEncryptionKey(getConf()))){
request.setSSECustomerKey(generateSSECustomerKey());
}
- ObjectMetadata meta = s3.getObjectMetadata(request);
+ ObjectMetadata meta = invoker.retryUntranslated("GET " + key, true,
+ () -> {
+ incrementStatistic(OBJECT_METADATA_REQUESTS);
+ return s3.getObjectMetadata(request);
+ });
incrementReadOperations();
return meta;
}
@@ -1048,40 +1206,68 @@ public class S3AFileSystem extends FileSystem {
/**
* Initiate a {@code listObjects} operation, incrementing metrics
* in the process.
+ *
+ * Retry policy: retry untranslated.
* @param request request to initiate
* @return the results
+ * @throws IOException if the retry invocation raises one (it shouldn't).
*/
- protected S3ListResult listObjects(S3ListRequest request) {
- incrementStatistic(OBJECT_LIST_REQUESTS);
+ @Retries.RetryRaw
+ protected S3ListResult listObjects(S3ListRequest request) throws IOException {
incrementReadOperations();
+ incrementStatistic(OBJECT_LIST_REQUESTS);
+ validateListArguments(request);
+ return invoker.retryUntranslated(
+ request.toString(),
+ true,
+ () -> {
+ if (useListV1) {
+ return S3ListResult.v1(s3.listObjects(request.getV1()));
+ } else {
+ return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ }
+ });
+ }
+
+ /**
+ * Validate the list arguments with this bucket's settings.
+ * @param request the request to validate
+ */
+ private void validateListArguments(S3ListRequest request) {
if (useListV1) {
Preconditions.checkArgument(request.isV1());
- return S3ListResult.v1(s3.listObjects(request.getV1()));
} else {
Preconditions.checkArgument(!request.isV1());
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
}
}
/**
* List the next set of objects.
+ * Retry policy: retry untranslated.
* @param request last list objects request to continue
* @param prevResult last paged result to continue from
* @return the next result object
+ * @throws IOException: none, just there for retryUntranslated.
*/
+ @Retries.RetryRaw
protected S3ListResult continueListObjects(S3ListRequest request,
- S3ListResult prevResult) {
- incrementStatistic(OBJECT_CONTINUE_LIST_REQUESTS);
+ S3ListResult prevResult) throws IOException {
incrementReadOperations();
- if (useListV1) {
- Preconditions.checkArgument(request.isV1());
- return S3ListResult.v1(s3.listNextBatchOfObjects(prevResult.getV1()));
- } else {
- Preconditions.checkArgument(!request.isV1());
- request.getV2().setContinuationToken(prevResult.getV2()
- .getNextContinuationToken());
- return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
- }
+ validateListArguments(request);
+ return invoker.retryUntranslated(
+ request.toString(),
+ true,
+ () -> {
+ incrementStatistic(OBJECT_CONTINUE_LIST_REQUESTS);
+ if (useListV1) {
+ return S3ListResult.v1(
+ s3.listNextBatchOfObjects(prevResult.getV1()));
+ } else {
+ request.getV2().setContinuationToken(prevResult.getV2()
+ .getNextContinuationToken());
+ return S3ListResult.v2(s3.listObjectsV2(request.getV2()));
+ }
+ });
}
/**
@@ -1101,16 +1287,52 @@ public class S3AFileSystem extends FileSystem {
}
/**
- * Delete an object.
+ * Delete an object. This is the low-level internal call which
+ * <i>does not</i> update the metastore.
* Increments the {@code OBJECT_DELETE_REQUESTS} and write
* operation statistics.
+ *
+ * Retry policy: retry untranslated; delete considered idempotent.
* @param key key to blob to delete.
+ * @throws AmazonClientException problems working with S3
+ * @throws InvalidRequestException if the request was rejected due to
+ * a mistaken attempt to delete the root directory.
*/
- private void deleteObject(String key) throws InvalidRequestException {
+ @VisibleForTesting
+ @Retries.RetryRaw
+ protected void deleteObject(String key)
+ throws AmazonClientException, IOException {
blockRootDelete(key);
incrementWriteOperations();
- incrementStatistic(OBJECT_DELETE_REQUESTS);
- s3.deleteObject(bucket, key);
+ invoker.retryUntranslated("Delete "+ bucket + ":/" + key,
+ DELETE_CONSIDERED_IDEMPOTENT,
+ ()-> {
+ incrementStatistic(OBJECT_DELETE_REQUESTS);
+ s3.deleteObject(bucket, key);
+ return null;
+ });
+ }
+
+ /**
+ * Delete an object, also updating the metastore.
+ * This call does <i>not</i> create any mock parent entries.
+ * Retry policy: retry untranslated; delete considered idempotent.
+ * @param f path path to delete
+ * @param key key of entry
+ * @param isFile is the path a file (used for instrumentation only)
+ * @throws AmazonClientException problems working with S3
+ * @throws IOException IO failure
+ */
+ @Retries.RetryRaw
+ void deleteObjectAtPath(Path f, String key, boolean isFile)
+ throws AmazonClientException, IOException {
+ if (isFile) {
+ instrumentation.fileDeleted(1);
+ } else {
+ instrumentation.directoryDeleted();
+ }
+ deleteObject(key);
+ metadataStore.delete(f);
}
/**
@@ -1130,17 +1352,23 @@ public class S3AFileSystem extends FileSystem {
* Perform a bulk object delete operation.
* Increments the {@code OBJECT_DELETE_REQUESTS} and write
* operation statistics.
+ * Retry policy: retry untranslated; delete considered idempotent.
* @param deleteRequest keys to delete on the s3-backend
* @throws MultiObjectDeleteException one or more of the keys could not
* be deleted.
* @throws AmazonClientException amazon-layer failure.
*/
+ @Retries.RetryRaw
private void deleteObjects(DeleteObjectsRequest deleteRequest)
- throws MultiObjectDeleteException, AmazonClientException {
+ throws MultiObjectDeleteException, AmazonClientException, IOException {
incrementWriteOperations();
- incrementStatistic(OBJECT_DELETE_REQUESTS, 1);
try {
- s3.deleteObjects(deleteRequest);
+ invoker.retryUntranslated("delete",
+ DELETE_CONSIDERED_IDEMPOTENT,
+ () -> {
+ incrementStatistic(OBJECT_DELETE_REQUESTS, 1);
+ return s3.deleteObjects(deleteRequest);
+ });
} catch (MultiObjectDeleteException e) {
// one or more of the operations failed.
List<MultiObjectDeleteException.DeleteError> errors = e.getErrors();
@@ -1185,6 +1413,7 @@ public class S3AFileSystem extends FileSystem {
ObjectMetadata metadata,
InputStream inputStream) {
Preconditions.checkNotNull(inputStream);
+ Preconditions.checkArgument(StringUtils.isNotEmpty(key), "Null/empty key");
PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, key,
inputStream, metadata);
setOptionalPutRequestParameters(putObjectRequest);
@@ -1231,36 +1460,32 @@ public class S3AFileSystem extends FileSystem {
* Because the operation is async, any stream supplied in the request
* must reference data (files, buffers) which stay valid until the upload
* completes.
+ * Retry policy: N/A: the transfer manager is performing the upload.
* @param putObjectRequest the request
* @return the upload initiated
*/
+ @Retries.OnceRaw
public UploadInfo putObject(PutObjectRequest putObjectRequest) {
- long len;
- if (putObjectRequest.getFile() != null) {
- len = putObjectRequest.getFile().length();
- } else {
- len = putObjectRequest.getMetadata().getContentLength();
- }
+ long len = getPutRequestLength(putObjectRequest);
+ LOG.debug("PUT {} bytes to {} via transfer manager ",
+ len, putObjectRequest.getKey());
incrementPutStartStatistics(len);
- try {
- Upload upload = transfers.upload(putObjectRequest);
- incrementPutCompletedStatistics(true, len);
- return new UploadInfo(upload, len);
- } catch (AmazonClientException e) {
- incrementPutCompletedStatistics(false, len);
- throw e;
- }
+ Upload upload = transfers.upload(putObjectRequest);
+ return new UploadInfo(upload, len);
}
/**
* PUT an object directly (i.e. not via the transfer manager).
* Byte length is calculated from the file length, or, if there is no
* file, from the content length of the header.
+ *
+ * Retry Policy: none.
* <i>Important: this call will close any input stream in the request.</i>
* @param putObjectRequest the request
* @return the upload initiated
* @throws AmazonClientException on problems
*/
+ @Retries.OnceRaw
PutObjectResult putObjectDirect(PutObjectRequest putObjectRequest)
throws AmazonClientException {
long len = getPutRequestLength(putObjectRequest);
@@ -1269,6 +1494,8 @@ public class S3AFileSystem extends FileSystem {
try {
PutObjectResult result = s3.putObject(putObjectRequest);
incrementPutCompletedStatistics(true, len);
+ // update metadata
+ finishedWrite(putObjectRequest.getKey(), len);
return result;
} catch (AmazonClientException e) {
incrementPutCompletedStatistics(false, len);
@@ -1297,11 +1524,14 @@ public class S3AFileSystem extends FileSystem {
* Upload part of a multi-partition file.
* Increments the write and put counters.
* <i>Important: this call does not close any input stream in the request.</i>
+ *
+ * Retry Policy: none.
* @param request request
* @return the result of the operation.
* @throws AmazonClientException on problems
*/
- public UploadPartResult uploadPart(UploadPartRequest request)
+ @Retries.OnceRaw
+ UploadPartResult uploadPart(UploadPartRequest request)
throws AmazonClientException {
long len = request.getPartSize();
incrementPutStartStatistics(len);
@@ -1366,7 +1596,7 @@ public class S3AFileSystem extends FileSystem {
/**
* A helper method to delete a list of keys on a s3-backend.
- *
+ * Retry policy: retry untranslated; delete considered idempotent.
* @param keysToDelete collection of keys to delete on the s3-backend.
* if empty, no request is made of the object store.
* @param clearKeys clears the keysToDelete-list after processing the list
@@ -1379,10 +1609,11 @@ public class S3AFileSystem extends FileSystem {
* @throws AmazonClientException amazon-layer failure.
*/
@VisibleForTesting
+ @Retries.RetryRaw
void removeKeys(List<DeleteObjectsRequest.KeyVersion> keysToDelete,
boolean clearKeys, boolean deleteFakeDir)
throws MultiObjectDeleteException, AmazonClientException,
- InvalidRequestException {
+ IOException {
if (keysToDelete.isEmpty()) {
// exit fast if there are no keys to delete
return;
@@ -1415,9 +1646,12 @@ public class S3AFileSystem extends FileSystem {
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
- * @return true if delete is successful else false.
+ * @return true if the path existed and then was deleted; false if there
+ * was no path in the first place, or the corner cases of root path deletion
+ * have surfaced.
* @throws IOException due to inability to delete a directory or file.
*/
+ @Retries.RetryTranslated
public boolean delete(Path f, boolean recursive) throws IOException {
try {
return innerDelete(innerGetFileStatus(f, true), recursive);
@@ -1437,14 +1671,15 @@ public class S3AFileSystem extends FileSystem {
* @param recursive if path is a directory and set to
* true, the directory is deleted else throws an exception. In
* case of a file the recursive can be set to either true or false.
- * @return true if delete is successful else false.
+ * @return true, except in the corner cases of root directory deletion
* @throws IOException due to inability to delete a directory or file.
* @throws AmazonClientException on failures inside the AWS SDK
*/
+ @Retries.RetryMixed
private boolean innerDelete(S3AFileStatus status, boolean recursive)
throws IOException, AmazonClientException {
Path f = status.getPath();
- LOG.debug("Delete path {} - recursive {}", f , recursive);
+ LOG.debug("Delete path {} - recursive {}", f, recursive);
String key = pathToKey(f);
@@ -1468,10 +1703,8 @@ public class S3AFileSystem extends FileSystem {
if (status.isEmptyDirectory() == Tristate.TRUE) {
LOG.debug("Deleting fake empty directory {}", key);
- // HADOOP-13761 S3Guard: retries here
- deleteObject(key);
- metadataStore.delete(f);
- instrumentation.directoryDeleted();
+ // HADOOP-13761 s3guard: retries here
+ deleteObjectAtPath(f, key, false);
} else {
LOG.debug("Getting objects for directory prefix {} to delete", key);
@@ -1486,7 +1719,6 @@ public class S3AFileSystem extends FileSystem {
LOG.debug("Got object to delete {}", summary.getKey());
if (keys.size() == MAX_ENTRIES_TO_DELETE) {
- // TODO: HADOOP-13761 S3Guard: retries
removeKeys(keys, true, false);
}
}
@@ -1505,15 +1737,10 @@ public class S3AFileSystem extends FileSystem {
metadataStore.deleteSubtree(f);
} else {
LOG.debug("delete: Path is a file");
- instrumentation.fileDeleted(1);
- deleteObject(key);
- metadataStore.delete(f);
+ deleteObjectAtPath(f, key, true);
}
- Path parent = f.getParent();
- if (parent != null) {
- createFakeDirectoryIfNecessary(parent);
- }
+ maybeCreateFakeParentDirectory(f);
return true;
}
@@ -1543,6 +1770,15 @@ public class S3AFileSystem extends FileSystem {
}
}
+ /**
+ * Create a fake directory if required.
+ * That is: it is not the root path and the path does not exist.
+ * Retry policy: retrying; untranslated.
+ * @param f path to create
+ * @throws IOException IO problem
+ * @throws AmazonClientException untranslated AWS client problem
+ */
+ @Retries.RetryTranslated
private void createFakeDirectoryIfNecessary(Path f)
throws IOException, AmazonClientException {
String key = pathToKey(f);
@@ -1553,6 +1789,21 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Create a fake parent directory if required.
+ * That is: it parent is not the root path and does not yet exist.
+ * @param path whose parent is created if needed.
+ * @throws IOException IO problem
+ * @throws AmazonClientException untranslated AWS client problem
+ */
+ void maybeCreateFakeParentDirectory(Path path)
+ throws IOException, AmazonClientException {
+ Path parent = path.getParent();
+ if (parent != null) {
+ createFakeDirectoryIfNecessary(parent);
+ }
+ }
+
+ /**
* List the statuses of the files/directories in the given path if the path is
* a directory.
*
@@ -1563,11 +1814,7 @@ public class S3AFileSystem extends FileSystem {
*/
public FileStatus[] listStatus(Path f) throws FileNotFoundException,
IOException {
- try {
- return innerListStatus(f);
- } catch (AmazonClientException e) {
- throw translateException("listStatus", f, e);
- }
+ return once("listStatus", f.toString(), () -> innerListStatus(f));
}
/**
@@ -1697,7 +1944,7 @@ public class S3AFileSystem extends FileSystem {
* Existence of the directory hierarchy is not an error.
* @param path path to create
* @param permission to apply to f
- * @return true if a directory was created
+ * @return true if a directory was created or already existed
* @throws FileAlreadyExistsException there is a file at the path specified
* @throws IOException other IO problems
*/
@@ -1787,6 +2034,7 @@ public class S3AFileSystem extends FileSystem {
* @throws FileNotFoundException when the path does not exist
* @throws IOException on other problems.
*/
+ @Retries.RetryTranslated
public FileStatus getFileStatus(final Path f) throws IOException {
return innerGetFileStatus(f, false);
}
@@ -1801,6 +2049,7 @@ public class S3AFileSystem extends FileSystem {
* @throws IOException on other problems.
*/
@VisibleForTesting
+ @Retries.RetryTranslated
S3AFileStatus innerGetFileStatus(final Path f,
boolean needEmptyDirectoryFlag) throws IOException {
incrementStatistic(INVOCATION_GET_FILE_STATUS);
@@ -1856,12 +2105,14 @@ public class S3AFileSystem extends FileSystem {
* Raw {@code getFileStatus} that talks direct to S3.
* Used to implement {@link #innerGetFileStatus(Path, boolean)},
* and for direct management of empty directory blobs.
+ * Retry policy: retry translated.
* @param path Qualified path
* @param key Key string for the path
* @return Status
* @throws FileNotFoundException when the path does not exist
* @throws IOException on other problems.
*/
+ @Retries.RetryTranslated
private S3AFileStatus s3GetFileStatus(final Path path, String key,
Set<Path> tombstones) throws IOException {
if (!key.isEmpty()) {
@@ -2000,8 +2251,11 @@ public class S3AFileSystem extends FileSystem {
/**
* Raw version of {@link FileSystem#exists(Path)} which uses S3 only:
* S3Guard MetadataStore, if any, will be skipped.
+ * Retry policy: retrying; translated.
* @return true if path exists in S3
+ * @throws IOException IO failure
*/
+ @Retries.RetryTranslated
private boolean s3Exists(final Path f) throws IOException {
Path path = qualify(f);
String key = pathToKey(path);
@@ -2033,12 +2287,7 @@ public class S3AFileSystem extends FileSystem {
@Override
public void copyFromLocalFile(boolean delSrc, boolean overwrite, Path src,
Path dst) throws IOException {
- try {
- innerCopyFromLocalFile(delSrc, overwrite, src, dst);
- } catch (AmazonClientException e) {
- throw translateException("copyFromLocalFile(" + src + ", " + dst + ")",
- src, e);
- }
+ innerCopyFromLocalFile(delSrc, overwrite, src, dst);
}
/**
@@ -2060,6 +2309,7 @@ public class S3AFileSystem extends FileSystem {
* @throws AmazonClientException failure in the AWS SDK
* @throws IllegalArgumentException if the source path is not on the local FS
*/
+ @Retries.RetryTranslated
private void innerCopyFromLocalFile(boolean delSrc, boolean overwrite,
Path src, Path dst)
throws IOException, FileAlreadyExistsException, AmazonClientException {
@@ -2089,25 +2339,66 @@ public class S3AFileSystem extends FileSystem {
}
final String key = pathToKey(dst);
final ObjectMetadata om = newObjectMetadata(srcfile.length());
+ Progressable progress = null;
PutObjectRequest putObjectRequest = newPutObjectRequest(key, om, srcfile);
+ invoker.retry("copyFromLocalFile(" + src + ")", dst.toString(), true,
+ () -> executePut(putObjectRequest, progress));
+ if (delSrc) {
+ local.delete(src, false);
+ }
+ }
+
+ /**
+ * Execute a PUT via the transfer manager, blocking for completion,
+ * updating the metastore afterwards.
+ * If the waiting for completion is interrupted, the upload will be
+ * aborted before an {@code InterruptedIOException} is thrown.
+ * @param putObjectRequest request
+ * @param progress optional progress callback
+ * @return the upload result
+ * @throws InterruptedIOException if the blocking was interrupted.
+ */
+ @Retries.OnceRaw
+ UploadResult executePut(PutObjectRequest putObjectRequest,
+ Progressable progress)
+ throws InterruptedIOException {
+ String key = putObjectRequest.getKey();
UploadInfo info = putObject(putObjectRequest);
Upload upload = info.getUpload();
ProgressableProgressListener listener = new ProgressableProgressListener(
- this, key, upload, null);
+ this, key, upload, progress);
upload.addProgressListener(listener);
- try {
- upload.waitForUploadResult();
- } catch (InterruptedException e) {
- throw new InterruptedIOException("Interrupted copying " + src
- + " to " + dst + ", cancelling");
- }
+ UploadResult result = waitForUploadCompletion(key, info);
listener.uploadCompleted();
-
- // This will delete unnecessary fake parent directories
+ // post-write actions
finishedWrite(key, info.getLength());
+ return result;
+ }
- if (delSrc) {
- local.delete(src, false);
+ /**
+ * Wait for an upload to complete.
+ * If the waiting for completion is interrupted, the upload will be
+ * aborted before an {@code InterruptedIOException} is thrown.
+ * @param upload upload to wait for
+ * @param key destination key
+ * @return the upload result
+ * @throws InterruptedIOException if the blocking was interrupted.
+ */
+ UploadResult waitForUploadCompletion(String key, UploadInfo uploadInfo)
+ throws InterruptedIOException {
+ Upload upload = uploadInfo.getUpload();
+ try {
+ UploadResult result = upload.waitForUploadResult();
+ incrementPutCompletedStatistics(true, uploadInfo.getLength());
+ return result;
+ } catch (InterruptedException e) {
+ LOG.info("Interrupted: aborting upload");
+ incrementPutCompletedStatistics(false, uploadInfo.getLength());
+ upload.abort();
+ throw (InterruptedIOException)
+ new InterruptedIOException("Interrupted in PUT to "
+ + keyToQualifiedPath(key))
+ .initCause(e);
}
}
@@ -2211,6 +2502,21 @@ public class S3AFileSystem extends FileSystem {
}
}
+ /**
+ * Initiate a multipart upload from the preconfigured request.
+ * Retry policy: none + untranslated.
+ * @param request request to initiate
+ * @return the result of the call
+ * @throws AmazonClientException on failures inside the AWS SDK
+ * @throws IOException Other IO problems
+ */
+ @Retries.OnceRaw
+ InitiateMultipartUploadResult initiateMultipartUpload(
+ InitiateMultipartUploadRequest request) throws IOException {
+ LOG.debug("Initiate multipart upload to {}", request.getKey());
+ incrementStatistic(OBJECT_MULTIPART_UPLOAD_INITIATED);
+ return getAmazonS3Client().initiateMultipartUpload(request);
+ }
protected void setOptionalCopyObjectRequestParameters(
CopyObjectRequest copyObjectRequest) throws IOException {
@@ -2286,6 +2592,7 @@ public class S3AFileSystem extends FileSystem {
* @param length total length of file written
*/
@InterfaceAudience.Private
+ @Retries.RetryTranslated("Exceptions are swallowed")
void finishedWrite(String key, long length) {
LOG.debug("Finished write to {}, len {}", key, length);
Path p = keyToQualifiedPath(key);
@@ -2310,9 +2617,10 @@ public class S3AFileSystem extends FileSystem {
/**
* Delete mock parent directories which are no longer needed.
- * This code swallows IO exceptions encountered
+ * Retry policy: retrying; exceptions swallowed.
* @param path path
*/
+ @Retries.RetryRaw("Exceptions are swallowed")
private void deleteUnnecessaryFakeDirectories(Path path) {
List<DeleteObjectsRequest.KeyVersion> keysToRemove = new ArrayList<>();
while (!path.isRoot()) {
@@ -2324,7 +2632,7 @@ public class S3AFileSystem extends FileSystem {
}
try {
removeKeys(keysToRemove, false, true);
- } catch(AmazonClientException | InvalidRequestException e) {
+ } catch(AmazonClientException | IOException e) {
instrumentation.errorIgnored();
if (LOG.isDebugEnabled()) {
StringBuilder sb = new StringBuilder();
@@ -2336,9 +2644,15 @@ public class S3AFileSystem extends FileSystem {
}
}
+ /**
+ * Create a fake directory, always ending in "/".
+ * Retry policy: retrying; translated.
+ * @param objectName name of directory object.
+ * @throws IOException IO failure
+ */
+ @Retries.RetryTranslated
private void createFakeDirectory(final String objectName)
- throws AmazonClientException, AmazonServiceException,
- InterruptedIOException {
+ throws IOException {
if (!objectName.endsWith("/")) {
createEmptyObject(objectName + "/");
} else {
@@ -2346,10 +2660,15 @@ public class S3AFileSystem extends FileSystem {
}
}
- // Used to create an empty file that represents an empty directory
+ /**
+ * Used to create an empty file that represents an empty directory.
+ * Retry policy: retrying; translated.
+ * @param objectName object to create
+ * @throws IOException IO failure
+ */
+ @Retries.RetryTranslated
private void createEmptyObject(final String objectName)
- throws AmazonClientException, AmazonServiceException,
- InterruptedIOException {
+ throws IOException {
final InputStream im = new InputStream() {
@Override
public int read() throws IOException {
@@ -2360,12 +2679,9 @@ public class S3AFileSystem extends FileSystem {
PutObjectRequest putObjectRequest = newPutObjectRequest(objectName,
newObjectMetadata(0L),
im);
- UploadInfo info = putObject(putObjectRequest);
- try {
- info.getUpload().waitForUploadResult();
- } catch (InterruptedException e) {
- throw new InterruptedIOException("Interrupted creating " + objectName);
- }
+ invoker.retry("PUT 0-byte object ", objectName,
+ true,
+ () -> putObjectDirect(putObjectRequest));
incrementPutProgressStatistics(objectName, 0);
instrumentation.directoryCreated();
}
@@ -2473,6 +2789,7 @@ public class S3AFileSystem extends FileSystem {
sb.append(", metastore=").append(metadataStore);
sb.append(", authoritative=").append(allowAuthoritative);
sb.append(", useListV1=").append(useListV1);
+ sb.append(", magicCommitter=").append(isMagicCommitEnabled());
sb.append(", boundedExecutor=").append(boundedThreadPool);
sb.append(", unboundedExecutor=").append(unboundedThreadPool);
sb.append(", statistics {")
@@ -2512,6 +2829,24 @@ public class S3AFileSystem extends FileSystem {
}
/**
+ * Is magic commit enabled?
+ * @return true if magic commit support is turned on.
+ */
+ public boolean isMagicCommitEnabled() {
+ return committerIntegration.isMagicCommitEnabled();
+ }
+
+ /**
+ * Predicate: is a path a magic commit path?
+ * True if magic commit is enabled and the path qualifies as special.
+ * @param path path to examine
+ * @return true if the path is or is under a magic directory
+ */
+ public boolean isMagicCommitPath(Path path) {
+ return committerIntegration.isMagicCommitPath(path);
+ }
+
+ /**
* Increments the statistic {@link Statistic#INVOCATION_GLOB_STATUS}.
* {@inheritDoc}
*/
@@ -2686,6 +3021,7 @@ public class S3AFileSystem extends FileSystem {
* @throws IOException if any I/O error occurred
*/
@Override
+ @Retries.OnceTranslated
public RemoteIterator<LocatedFileStatus> listLocatedStatus(final Path f,
final PathFilter filter)
throws FileNotFoundException, IOException {
@@ -2738,203 +3074,91 @@ public class S3AFileSystem extends FileSystem {
}
/**
- * Helper for an ongoing write operation.
- * <p>
- * It hides direct access to the S3 API from the output stream,
- * and is a location where the object upload process can be evolved/enhanced.
- * <p>
- * Features
- * <ul>
- * <li>Methods to create and submit requests to S3, so avoiding
- * all direct interaction with the AWS APIs.</li>
- * <li>Some extra preflight checks of arguments, so failing fast on
- * errors.</li>
- * <li>Callbacks to let the FS know of events in the output stream
- * upload process.</li>
- * </ul>
- *
- * Each instance of this state is unique to a single output stream.
- */
- final class WriteOperationHelper {
- private final String key;
-
- private WriteOperationHelper(String key) {
- this.key = key;
- }
-
- /**
- * Create a {@link PutObjectRequest} request.
- * If {@code length} is set, the metadata is configured with the size of
- * the upload.
- * @param inputStream source data.
- * @param length size, if known. Use -1 for not known
- * @return the request
- */
- PutObjectRequest newPutRequest(InputStream inputStream, long length) {
- PutObjectRequest request = newPutObjectRequest(key,
- newObjectMetadata(length), inputStream);
- return request;
- }
-
- /**
- * Create a {@link PutObjectRequest} request to upload a file.
- * @param sourceFile source file
- * @return the request
- */
- PutObjectRequest newPutRequest(File sourceFile) {
- int length = (int) sourceFile.length();
- PutObjectRequest request = newPutObjectRequest(key,
- newObjectMetadata(length), sourceFile);
- return request;
- }
-
- /**
- * Callback on a successful write.
- */
- void writeSuccessful(long length) {
- finishedWrite(key, length);
- }
-
- /**
- * Callback on a write failure.
- * @param e Any exception raised which triggered the failure.
- */
- void writeFailed(Exception e) {
- LOG.debug("Write to {} failed", this, e);
- }
-
- /**
- * Create a new object metadata instance.
- * Any standard metadata headers are added here, for example:
- * encryption.
- * @param length size, if known. Use -1 for not known
- * @return a new metadata instance
- */
- public ObjectMetadata newObjectMetadata(long length) {
- return S3AFileSystem.this.newObjectMetadata(length);
- }
-
- /**
- * Start the multipart upload process.
- * @return the upload result containing the ID
- * @throws IOException IO problem
- */
- String initiateMultiPartUpload() throws IOException {
- LOG.debug("Initiating Multipart upload");
- final InitiateMultipartUploadRequest initiateMPURequest =
- new InitiateMultipartUploadRequest(bucket,
- key,
- newObjectMetadata(-1));
- initiateMPURequest.setCannedACL(cannedACL);
- setOptionalMultipartUploadRequestParameters(initiateMPURequest);
- try {
- return s3.initiateMultipartUpload(initiateMPURequest)
- .getUploadId();
- } catch (AmazonClientException ace) {
- throw translateException("initiate MultiPartUpload", key, ace);
+ * Listing all multipart uploads; limited to the first few hundred.
+ * Retry policy: retry, translated.
+ * @return a listing of multipart uploads.
+ * @param prefix prefix to scan for, "" for none
+ * @throws IOException IO failure, including any uprated AmazonClientException
+ */
+ @InterfaceAudience.Private
+ @Retries.RetryTranslated
+ public List<MultipartUpload> listMultipartUploads(String prefix)
+ throws IOException {
+ ListMultipartUploadsRequest request = new ListMultipartUploadsRequest(
+ bucket);
+ if (!prefix.isEmpty()) {
+ if (!prefix.endsWith("/")) {
+ prefix = prefix + "/";
}
+ request.setPrefix(prefix);
}
- /**
- * Complete a multipart upload operation.
- * @param uploadId multipart operation Id
- * @param partETags list of partial uploads
- * @return the result
- * @throws AmazonClientException on problems.
- */
- CompleteMultipartUploadResult completeMultipartUpload(String uploadId,
- List<PartETag> partETags) throws AmazonClientException {
- Preconditions.checkNotNull(uploadId);
- Preconditions.checkNotNull(partETags);
- Preconditions.checkArgument(!partETags.isEmpty(),
- "No partitions have been uploaded");
- LOG.debug("Completing multipart upload {} with {} parts",
- uploadId, partETags.size());
- // a copy of the list is required, so that the AWS SDK doesn't
- // attempt to sort an unmodifiable list.
- return s3.completeMultipartUpload(
- new CompleteMultipartUploadRequest(bucket,
- key,
- uploadId,
- new ArrayList<>(partETags)));
- }
-
- /**
- * Abort a multipart upload operation.
- * @param uploadId multipart operation Id
- * @throws AmazonClientException on problems.
- */
- void abortMultipartUpload(String uploadId) throws AmazonClientException {
- LOG.debug("Aborting multipart upload {}", uploadId);
- s3.abortMultipartUpload(
- new AbortMultipartUploadRequest(bucket, key, uploadId));
- }
-
- /**
- * Create and initialize a part request of a multipart upload.
- * Exactly one of: {@code uploadStream} or {@code sourceFile}
- * must be specified.
- * @param uploadId ID of ongoing upload
- * @param partNumber current part number of the upload
- * @param size amount of data
- * @param uploadStream source of data to upload
- * @param sourceFile optional source file.
- * @return the request.
- */
- UploadPartRequest newUploadPartRequest(String uploadId,
- int partNumber, int size, InputStream uploadStream, File sourceFile) {
- Preconditions.checkNotNull(uploadId);
- // exactly one source must be set; xor verifies this
- Preconditions.checkArgument((uploadStream != null) ^ (sourceFile != null),
- "Data source");
- Preconditions.checkArgument(size > 0, "Invalid partition size %s", size);
- Preconditions.checkArgument(partNumber > 0 && partNumber <= 10000,
- "partNumber must be between 1 and 10000 inclusive, but is %s",
- partNumber);
-
- LOG.debug("Creating part upload request for {} #{} size {}",
- uploadId, partNumber, size);
- UploadPartRequest request = new UploadPartRequest()
- .withBucketName(bucket)
- .withKey(key)
- .withUploadId(uploadId)
- .withPartNumber(partNumber)
- .withPartSize(size);
- if (uploadStream != null) {
- // there's an upload stream. Bind to it.
- request.setInputStream(uploadStream);
- } else {
- request.setFile(sourceFile);
- }
- return request;
- }
-
- /**
- * The toString method is intended to be used in logging/toString calls.
- * @return a string description.
- */
- @Override
- public String toString() {
- final StringBuilder sb = new StringBuilder(
- "{bucket=").append(bucket);
- sb.append(", key='").append(key).append('\'');
- sb.append('}');
- return sb.toString();
- }
-
- /**
- * PUT an object directly (i.e. not via the transfer manager).
- * @param putObjectRequest the request
- * @return the upload initiated
- * @throws IOException on problems
- */
- PutObjectResult putObject(PutObjectRequest putObjectRequest)
- throws IOException {
- try {
- return putObjectDirect(putObjectRequest);
- } catch (AmazonClientException e) {
- throw translateException("put", putObjectRequest.getKey(), e);
- }
+ return invoker.retry("listMultipartUploads", prefix, true,
+ () -> s3.listMultipartUploads(request).getMultipartUploads());
+ }
+
+ /**
+ * Abort a multipart upload.
+ * Retry policy: none.
+ * @param destKey destination key
+ * @param uploadId Upload ID
+ */
+ @Retries.OnceRaw
+ void abortMultipartUpload(String destKey, String uploadId) {
+ LOG.info("Aborting multipart upload {} to {}", uploadId, destKey);
+ getAmazonS3Client().abortMultipartUpload(
+ new AbortMultipartUploadRequest(getBucket(),
+ destKey,
+ uploadId));
+ }
+
+ /**
+ * Abort a multipart upload.
+ * Retry policy: none.
+ * @param upload the listed upload to abort.
+ */
+ @Retries.OnceRaw
+ void abortMultipartUpload(MultipartUpload upload) {
+ String destKey;
+ String uploadId;
+ destKey = upload.getKey();
+ uploadId = upload.getUploadId();
+ if (LOG.isInfoEnabled()) {
+ DateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+ LOG.info("Aborting multipart upload {} to {} initiated by {} on {}",
+ uploadId, destKey, upload.getInitiator(),
+ df.format(upload.getInitiated()));
+ }
+ getAmazonS3Client().abortMultipartUpload(
+ new AbortMultipartUploadRequest(getBucket(),
+ destKey,
+ uploadId));
+ }
+
+ /**
+ * Create a new instance of the committer statistics.
+ * @return a new committer statistics instance
+ */
+ public S3AInstrumentation.CommitterStatistics newCommitterStatistics() {
+ return instrumentation.newCommitterStatistics();
+ }
+
+ /**
+ * Return the capabilities of this filesystem instance.
+ * @param capability string to query the stream support for.
+ * @return whether the FS instance has the capability.
+ */
+ @Override
+ public boolean hasCapability(String capability) {
+
+ switch (capability.toLowerCase(Locale.ENGLISH)) {
+
+ case CommitConstants.STORE_CAPABILITY_MAGIC_COMMITTER:
+ // capability depends on FS configuration
+ return isMagicCommitEnabled();
+
+ default:
+ return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 94d7701..7e6d640 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.fs.s3a;
-import com.amazonaws.AmazonClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.SSECustomerKey;
import com.google.common.base.Preconditions;
@@ -39,7 +39,6 @@ import java.io.EOFException;
import java.io.IOException;
import static org.apache.commons.lang3.StringUtils.isNotEmpty;
-import static org.apache.hadoop.fs.s3a.S3AUtils.*;
/**
* The input stream for an S3A object.
@@ -86,6 +85,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
private String serverSideEncryptionKey;
private final S3AInputPolicy inputPolicy;
private long readahead = Constants.DEFAULT_READAHEAD_RANGE;
+ private final Invoker invoker;
/**
* This is the actual position within the object, used by
@@ -104,14 +104,29 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
*/
private long contentRangeStart;
+ /**
+ * Create the stream.
+ * This does not attempt to open it; that is only done on the first
+ * actual read() operation.
+ * @param s3Attributes object attributes from a HEAD request
+ * @param contentLength length of content
+ * @param client S3 client to use
+ * @param stats statistics to update
+ * @param instrumentation instrumentation to update
+ * @param readahead readahead bytes
+ * @param inputPolicy IO policy
+ * @param invoker preconfigured invoker
+ */
public S3AInputStream(S3ObjectAttributes s3Attributes,
long contentLength,
AmazonS3 client,
FileSystem.Statistics stats,
S3AInstrumentation instrumentation,
long readahead,
- S3AInputPolicy inputPolicy) {
- Preconditions.checkArgument(isNotEmpty(s3Attributes.getBucket()), "No Bucket");
+ S3AInputPolicy inputPolicy,
+ Invoker invoker) {
+ Preconditions.checkArgument(isNotEmpty(s3Attributes.getBucket()),
+ "No Bucket");
Preconditions.checkArgument(isNotEmpty(s3Attributes.getKey()), "No Key");
Preconditions.checkArgument(contentLength >= 0, "Negative content length");
this.bucket = s3Attributes.getBucket();
@@ -126,6 +141,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
this.serverSideEncryptionKey = s3Attributes.getServerSideEncryptionKey();
this.inputPolicy = inputPolicy;
setReadahead(readahead);
+ this.invoker = invoker;
}
/**
@@ -149,22 +165,22 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
" streamPosition={}, nextReadPosition={}",
uri, reason, targetPos, contentRangeFinish, length, pos, nextReadPos);
- streamStatistics.streamOpened();
- try {
- GetObjectRequest request = new GetObjectRequest(bucket, key)
- .withRange(targetPos, contentRangeFinish - 1);
- if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) &&
- StringUtils.isNotBlank(serverSideEncryptionKey)){
- request.setSSECustomerKey(new SSECustomerKey(serverSideEncryptionKey));
- }
- wrappedStream = client.getObject(request).getObjectContent();
- contentRangeStart = targetPos;
- if (wrappedStream == null) {
- throw new IOException("Null IO stream from reopen of (" + reason + ") "
- + uri);
- }
- } catch (AmazonClientException e) {
- throw translateException("Reopen at position " + targetPos, uri, e);
+ long opencount = streamStatistics.streamOpened();
+ GetObjectRequest request = new GetObjectRequest(bucket, key)
+ .withRange(targetPos, contentRangeFinish - 1);
+ if (S3AEncryptionMethods.SSE_C.equals(serverSideEncryptionAlgorithm) &&
+ StringUtils.isNotBlank(serverSideEncryptionKey)){
+ request.setSSECustomerKey(new SSECustomerKey(serverSideEncryptionKey));
+ }
+ String text = String.format("Failed to %s %s at %d",
+ (opencount == 0 ? "open" : "re-open"), uri, targetPos);
+ S3Object object = invoker.retry(text, uri, true,
+ () -> client.getObject(request));
+ wrappedStream = object.getObjectContent();
+ contentRangeStart = targetPos;
+ if (wrappedStream == null) {
+ throw new IOException("Null IO stream from reopen of (" + reason + ") "
+ + uri);
}
this.pos = targetPos;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/49] hadoop git commit: YARN-7529.
TestYarnNativeServices#testRecoverComponentsAfterRMRestart() fails
intermittently. Contributed by Chandni Singh
Posted by su...@apache.org.
YARN-7529. TestYarnNativeServices#testRecoverComponentsAfterRMRestart() fails intermittently. Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f9d7a14
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f9d7a14
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f9d7a14
Branch: refs/heads/YARN-5881
Commit: 6f9d7a146d5940a9e8a7913c19b43b265d6bfa32
Parents: 6903cf0
Author: Billie Rinaldi <bi...@apache.org>
Authored: Mon Nov 20 07:37:04 2017 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Mon Nov 20 07:37:04 2017 -0800
----------------------------------------------------------------------
.../yarn/service/TestYarnNativeServices.java | 42 +++++++++-----------
1 file changed, 18 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f9d7a14/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index f98d90a..1c517d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -176,7 +176,8 @@ public class TestYarnNativeServices extends ServiceTestUtils {
ServiceClient client = createClient();
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
- waitForAllCompToBeReady(client, exampleApp);
+ Multimap<String, String> containersBeforeFailure =
+ waitForAllCompToBeReady(client, exampleApp);
LOG.info("Restart the resource manager");
getYarnCluster().restartResourceManager(
@@ -191,9 +192,6 @@ public class TestYarnNativeServices extends ServiceTestUtils {
ApplicationAttemptId applicationAttemptId = client.getYarnClient()
.getApplicationReport(exampleAppId).getCurrentApplicationAttemptId();
- Multimap<String, String> containersBeforeFailure = getContainersForAllComp(
- client, exampleApp);
-
LOG.info("Fail the application attempt {}", applicationAttemptId);
client.getYarnClient().failApplicationAttempt(applicationAttemptId);
//wait until attempt 2 is running
@@ -208,7 +206,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}
}, 2000, 200000);
- Multimap<String, String> containersAfterFailure = getContainersForAllComp(
+ Multimap<String, String> containersAfterFailure = waitForAllCompToBeReady(
client, exampleApp);
Assert.assertEquals("component container affected by restart",
containersBeforeFailure, containersAfterFailure);
@@ -318,14 +316,26 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}, 2000, 200000);
}
- // wait until all the containers for all components become ready state
- private void waitForAllCompToBeReady(ServiceClient client,
+ /**
+ * Wait until all the containers for all components become ready state.
+ *
+ * @param client
+ * @param exampleApp
+ * @return all ready containers of a service.
+ * @throws TimeoutException
+ * @throws InterruptedException
+ */
+ private Multimap<String, String> waitForAllCompToBeReady(ServiceClient client,
Service exampleApp) throws TimeoutException, InterruptedException {
int expectedTotalContainers = countTotalContainers(exampleApp);
+
+ Multimap<String, String> allContainers = HashMultimap.create();
+
GenericTestUtils.waitFor(() -> {
try {
Service retrievedApp = client.getStatus(exampleApp.getName());
int totalReadyContainers = 0;
+ allContainers.clear();
LOG.info("Num Components " + retrievedApp.getComponents().size());
for (Component component : retrievedApp.getComponents()) {
LOG.info("looking for " + component.getName());
@@ -339,6 +349,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
+ component.getName());
if (container.getState() == ContainerState.READY) {
totalReadyContainers++;
+ allContainers.put(component.getName(), container.getId());
LOG.info("Found 1 ready container " + container.getId());
}
}
@@ -358,23 +369,6 @@ public class TestYarnNativeServices extends ServiceTestUtils {
return false;
}
}, 2000, 200000);
- }
-
- /**
- * Get all containers of a service.
- */
- private Multimap<String, String> getContainersForAllComp(ServiceClient client,
- Service example) throws IOException, YarnException {
-
- Multimap<String, String> allContainers = HashMultimap.create();
- Service retrievedApp = client.getStatus(example.getName());
- retrievedApp.getComponents().forEach(component -> {
- if (component.getContainers() != null) {
- component.getContainers().forEach(container -> {
- allContainers.put(component.getName(), container.getId());
- });
- }
- });
return allContainers;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/49] hadoop git commit: YARN-7527. Over-allocate node resource in
async-scheduling mode of CapacityScheduler. (Tao Yang via wangda)
Posted by su...@apache.org.
YARN-7527. Over-allocate node resource in async-scheduling mode of CapacityScheduler. (Tao Yang via wangda)
Change-Id: I51ae6c2ab7a3d1febdd7d8d0519b63a13295ac7d
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d781dd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d781dd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d781dd0
Branch: refs/heads/YARN-5881
Commit: 0d781dd03b979d65de94978071b2faa55005b34a
Parents: c326fc8
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Nov 20 11:48:15 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Nov 20 11:48:15 2017 -0800
----------------------------------------------------------------------
.../scheduler/common/fica/FiCaSchedulerApp.java | 4 +-
.../TestCapacitySchedulerAsyncScheduling.java | 71 ++++++++++++++++++++
2 files changed, 74 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d781dd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 40405fc..e9bee14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -417,7 +417,9 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Common part of check container allocation regardless if it is a
// increase container or regular container
- commonCheckContainerAllocation(allocation, schedulerContainer);
+ if (!commonCheckContainerAllocation(allocation, schedulerContainer)) {
+ return false;
+ }
} else {
// Container reserved first time will be NEW, after the container
// accepted & confirmed, it will become RESERVED state
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d781dd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
index 0c3130d..77596e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAsyncScheduling.java
@@ -405,6 +405,77 @@ public class TestCapacitySchedulerAsyncScheduling {
rm.stop();
}
+ @Test (timeout = 30000)
+ public void testNodeResourceOverAllocated()
+ throws Exception {
+ // disable async-scheduling for simulating complex scene
+ Configuration disableAsyncConf = new Configuration(conf);
+ disableAsyncConf.setBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, false);
+
+ // init RM & NMs & Nodes
+ final MockRM rm = new MockRM(disableAsyncConf);
+ rm.start();
+ final MockNM nm1 = rm.registerNode("h1:1234", 9 * GB);
+ final MockNM nm2 = rm.registerNode("h2:1234", 9 * GB);
+ List<MockNM> nmLst = new ArrayList<>();
+ nmLst.add(nm1);
+ nmLst.add(nm2);
+
+ // init scheduler & nodes
+ while (
+ ((CapacityScheduler) rm.getRMContext().getScheduler()).getNodeTracker()
+ .nodeCount() < 2) {
+ Thread.sleep(10);
+ }
+ Assert.assertEquals(2,
+ ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
+ .getNodeTracker().nodeCount());
+ CapacityScheduler scheduler =
+ (CapacityScheduler) rm.getRMContext().getScheduler();
+ SchedulerNode sn1 = scheduler.getSchedulerNode(nm1.getNodeId());
+
+ // launch app
+ RMApp app = rm.submitApp(200, "app", "user", null, false, "default",
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS, null, null, true, true);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
+ FiCaSchedulerApp schedulerApp =
+ scheduler.getApplicationAttempt(am.getApplicationAttemptId());
+ // allocate 2 containers and running on nm1
+ Resource containerResource = Resources.createResource(5 * GB);
+ am.allocate(Arrays.asList(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*", containerResource, 2)),
+ null);
+
+ // generate over-allocated proposals for nm1
+ for (int containerNo = 2; containerNo <= 3; containerNo++) {
+ Container container = Container.newInstance(
+ ContainerId.newContainerId(am.getApplicationAttemptId(), containerNo),
+ sn1.getNodeID(), sn1.getHttpAddress(), containerResource,
+ Priority.newInstance(0), null);
+ RMContainer rmContainer = new RMContainerImpl(container,
+ SchedulerRequestKey.create(ResourceRequest
+ .newInstance(Priority.newInstance(0), "*", containerResource, 1)),
+ am.getApplicationAttemptId(), sn1.getNodeID(), "user",
+ rm.getRMContext());
+ SchedulerContainer newContainer = new SchedulerContainer(schedulerApp,
+ scheduler.getNode(sn1.getNodeID()), rmContainer, "", true);
+ ContainerAllocationProposal newContainerProposal =
+ new ContainerAllocationProposal(newContainer, null, null,
+ NodeType.OFF_SWITCH, NodeType.OFF_SWITCH,
+ SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY, containerResource);
+ List<ContainerAllocationProposal> newProposals = new ArrayList<>();
+ newProposals.add(newContainerProposal);
+ ResourceCommitRequest request =
+ new ResourceCommitRequest(newProposals, null, null);
+ scheduler.tryCommit(scheduler.getClusterResource(), request);
+ }
+ // make sure node resource can't be over-allocated!
+ Assert.assertTrue("Node resource is Over-allocated!",
+ sn1.getUnallocatedResource().getMemorySize() > 0);
+ rm.stop();
+ }
+
private void allocateAndLaunchContainers(MockAM am, MockNM nm, MockRM rm,
int nContainer, Resource resource, int priority, int startContainerId)
throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/49] hadoop git commit: YARN-7411. Inter-Queue preemption's
computeFixpointAllocation need to handle absolute resources while computing
normalizedGuarantee. (Sunil G via wangda)
Posted by su...@apache.org.
YARN-7411. Inter-Queue preemption's computeFixpointAllocation need to handle absolute resources while computing normalizedGuarantee. (Sunil G via wangda)
Change-Id: I41b1d7558c20fc4eb2050d40134175a2ef6330cb
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0238ca3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0238ca3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0238ca3
Branch: refs/heads/YARN-5881
Commit: a0238ca39a0880eebb33bc409ad0fd603f4305ad
Parents: f0d91df
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Nov 13 16:26:27 2017 -0800
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../api/records/impl/pb/ResourcePBImpl.java | 12 ----
.../resource/DefaultResourceCalculator.java | 8 +++
.../resource/DominantResourceCalculator.java | 21 ++++++
.../yarn/util/resource/ResourceCalculator.java | 14 +++-
.../hadoop/yarn/util/resource/Resources.java | 5 ++
.../AbstractPreemptableResourceCalculator.java | 24 ++++++-
.../monitor/capacity/TempQueuePerPartition.java | 12 ++--
...alCapacityPreemptionPolicyMockFramework.java | 14 ++++
...pacityPreemptionPolicyForNodePartitions.java | 76 ++++++++++++++++++++
9 files changed, 166 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 401e0c0..4f90133 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
@@ -152,17 +151,6 @@ public class ResourcePBImpl extends Resource {
.newInstance(ResourceInformation.VCORES);
this.setMemorySize(p.getMemory());
this.setVirtualCores(p.getVirtualCores());
-
- // Update missing resource information on respective index.
- updateResourceInformationMap(types);
- }
-
- private void updateResourceInformationMap(ResourceInformation[] types) {
- for (int i = 0; i < types.length; i++) {
- if (resources[i] == null) {
- resources[i] = ResourceInformation.newInstance(types[i]);
- }
- }
}
private static ResourceInformation newDefaultInformation(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
index aefa85c..6375c4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
@@ -112,6 +112,14 @@ public class DefaultResourceCalculator extends ResourceCalculator {
}
@Override
+ public Resource multiplyAndNormalizeUp(Resource r, double[] by,
+ Resource stepFactor) {
+ return Resources.createResource(
+ roundUp((long) (r.getMemorySize() * by[0] + 0.5),
+ stepFactor.getMemorySize()));
+ }
+
+ @Override
public Resource multiplyAndNormalizeDown(Resource r, double by,
Resource stepFactor) {
return Resources.createResource(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index cc4acf6..26ddc18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -496,6 +496,27 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
@Override
+ public Resource multiplyAndNormalizeUp(Resource r, double[] by,
+ Resource stepFactor) {
+ Resource ret = Resource.newInstance(r);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation rResourceInformation = r.getResourceInformation(i);
+ ResourceInformation stepFactorResourceInformation = stepFactor
+ .getResourceInformation(i);
+
+ long rValue = rResourceInformation.getValue();
+ long stepFactorValue = UnitsConversionUtil.convert(
+ stepFactorResourceInformation.getUnits(),
+ rResourceInformation.getUnits(),
+ stepFactorResourceInformation.getValue());
+ ret.setResourceValue(i, ResourceCalculator
+ .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue));
+ }
+ return ret;
+ }
+
+ @Override
public Resource multiplyAndNormalizeUp(Resource r, double by,
Resource stepFactor) {
return this.multiplyAndNormalize(r, by, stepFactor, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index a816290..dfd6684 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -125,7 +125,19 @@ public abstract class ResourceCalculator {
*/
public abstract Resource multiplyAndNormalizeUp(
Resource r, double by, Resource stepFactor);
-
+
+ /**
+ * Multiply resource <code>r</code> by factor <code>by</code>
+ * and normalize up using step-factor <code>stepFactor</code>.
+ *
+ * @param r resource to be multiplied
+ * @param by multiplier array for all resource types
+ * @param stepFactor factor by which to normalize up
+ * @return resulting normalized resource
+ */
+ public abstract Resource multiplyAndNormalizeUp(
+ Resource r, double[] by, Resource stepFactor);
+
/**
* Multiply resource <code>r</code> by factor <code>by</code>
* and normalize down using step-factor <code>stepFactor</code>.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 0426ab7..1c08844 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -347,6 +347,11 @@ public class Resources {
return lhs;
}
+ public static Resource multiplyAndNormalizeUp(ResourceCalculator calculator,
+ Resource lhs, double[] by, Resource factor) {
+ return calculator.multiplyAndNormalizeUp(lhs, by, factor);
+ }
+
public static Resource multiplyAndNormalizeUp(
ResourceCalculator calculator,Resource lhs, double by, Resource factor) {
return calculator.multiplyAndNormalizeUp(lhs, by, factor);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
index a80f317..5196831 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java
@@ -19,8 +19,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.PriorityUtilizationQueueOrderingPolicy;
+import org.apache.hadoop.yarn.util.UnitsConversionUtil;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
@@ -198,18 +201,33 @@ public class AbstractPreemptableResourceCalculator {
private void resetCapacity(Resource clusterResource,
Collection<TempQueuePerPartition> queues, boolean ignoreGuar) {
Resource activeCap = Resource.newInstance(0, 0);
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
if (ignoreGuar) {
for (TempQueuePerPartition q : queues) {
- q.normalizedGuarantee = 1.0f / queues.size();
+ for (int i = 0; i < maxLength; i++) {
+ q.normalizedGuarantee[i] = 1.0f / queues.size();
+ }
}
} else {
for (TempQueuePerPartition q : queues) {
Resources.addTo(activeCap, q.getGuaranteed());
}
for (TempQueuePerPartition q : queues) {
- q.normalizedGuarantee = Resources.divide(rc, clusterResource,
- q.getGuaranteed(), activeCap);
+ for (int i = 0; i < maxLength; i++) {
+ ResourceInformation nResourceInformation = q.getGuaranteed()
+ .getResourceInformation(i);
+ ResourceInformation dResourceInformation = activeCap
+ .getResourceInformation(i);
+
+ long nValue = nResourceInformation.getValue();
+ long dValue = UnitsConversionUtil.convert(
+ dResourceInformation.getUnits(), nResourceInformation.getUnits(),
+ dResourceInformation.getValue());
+ if (dValue != 0) {
+ q.normalizedGuarantee[i] = (float) nValue / dValue;
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
index bd236fe..4d71223 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java
@@ -22,9 +22,11 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -46,7 +48,7 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
Resource untouchableExtra;
Resource preemptableExtra;
- double normalizedGuarantee;
+ double[] normalizedGuarantee;
private Resource effMinRes;
private Resource effMaxRes;
@@ -88,7 +90,8 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
pendingDeductReserved = Resources.createResource(0);
}
- this.normalizedGuarantee = Float.NaN;
+ this.normalizedGuarantee = new double[ResourceUtils
+ .getNumberOfKnownResourceTypes()];
this.children = new ArrayList<>();
this.apps = new ArrayList<>();
this.untouchableExtra = Resource.newInstance(0, 0);
@@ -240,8 +243,9 @@ public class TempQueuePerPartition extends AbstractPreemptionEntity {
sb.append(" NAME: " + queueName).append(" CUR: ").append(current)
.append(" PEN: ").append(pending).append(" RESERVED: ").append(reserved)
.append(" GAR: ").append(getGuaranteed()).append(" NORM: ")
- .append(normalizedGuarantee).append(" IDEAL_ASSIGNED: ")
- .append(idealAssigned).append(" IDEAL_PREEMPT: ").append(toBePreempted)
+ .append(Arrays.toString(normalizedGuarantee))
+ .append(" IDEAL_ASSIGNED: ").append(idealAssigned)
+ .append(" IDEAL_PREEMPT: ").append(toBePreempted)
.append(" ACTUAL_PREEMPT: ").append(getActuallyToBePreempted())
.append(" UNTOUCHABLE: ").append(untouchableExtra)
.append(" PREEMPTABLE: ").append(preemptableExtra).append("\n");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 1dda8a6..ca43a95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
@@ -533,6 +535,18 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
} else {
res = Resources.createResource(Integer.valueOf(resource[0]),
Integer.valueOf(resource[1]));
+ if (resource.length > 2) {
+ // Using the same order of resources from ResourceUtils, set resource
+ // informations.
+ ResourceInformation[] storedResourceInfo = ResourceUtils
+ .getResourceTypesArray();
+ for (int i = 2; i < resource.length; i++) {
+ res.setResourceInformation(storedResourceInfo[i].getName(),
+ ResourceInformation.newInstance(storedResourceInfo[i].getName(),
+ storedResourceInfo[i].getUnits(),
+ Integer.valueOf(resource[i])));
+ }
+ }
}
return res;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0238ca3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
index 1fd455a..67c09cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java
@@ -18,11 +18,17 @@
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.never;
@@ -613,4 +619,74 @@ public class TestProportionalCapacityPreemptionPolicyForNodePartitions
verify(mDisp, never()).handle(
argThat(new IsPreemptionRequestFor(getAppAttemptId(3))));
}
+
+ @Test
+ public void testNormalizeGuaranteeWithMultipleResource() throws IOException {
+ // Initialize resource map
+ Map<String, ResourceInformation> riMap = new HashMap<>();
+ String RESOURCE_1 = "res1";
+
+ // Initialize mandatory resources
+ ResourceInformation memory = ResourceInformation.newInstance(
+ ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.MEMORY_MB.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+ ResourceInformation vcores = ResourceInformation.newInstance(
+ ResourceInformation.VCORES.getName(),
+ ResourceInformation.VCORES.getUnits(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+ riMap.put(ResourceInformation.MEMORY_URI, memory);
+ riMap.put(ResourceInformation.VCORES_URI, vcores);
+ riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0,
+ ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE));
+
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+
+ /**
+ * Queue structure is:
+ *
+ * <pre>
+ * root
+ * / \
+ * a b
+ * / \ / \
+ * a1 a2 b1 b2
+ * </pre>
+ *
+ * a1 and b2 are using most of resources.
+ * a2 and b1 needs more resources. Both are under served.
+ * hence demand will consider both queue's need while trying to
+ * do preemption.
+ */
+ String labelsConfig =
+ "=100,true;";
+ String nodesConfig =
+ "n1=;"; // n1 is default partition
+ String queuesConfig =
+ // guaranteed,max,used,pending
+ "root(=[100:100:10 100:100:10 100:100:10 100:100:10]);" + //root
+ "-a(=[50:80:4 100:100:10 80:90:10 30:20:4]);" + // a
+ "--a1(=[25:30:2 100:50:10 80:90:10 0]);" + // a1
+ "--a2(=[25:50:2 100:50:10 0 30:20:4]);" + // a2
+ "-b(=[50:20:6 100:100:10 20:10 40:50:8]);" + // b
+ "--b1(=[25:5:4 100:20:10 0 20:10:4]);" + // b1
+ "--b2(=[25:15:2 100:20:10 20:10 20:10:4])"; // b2
+ String appsConfig=
+ //queueName\t(priority,resource,host,expression,#repeat,reserved)
+ "a1\t" // app1 in a1
+ + "(1,8:9:1,n1,,10,false);" +
+ "b2\t" // app2 in b2
+ + "(1,2:1,n1,,10,false)"; // 80 of y
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ verify(mDisp, times(7)).handle(
+ argThat(new IsPreemptionRequestFor(getAppAttemptId(1))));
+
+ riMap.remove(RESOURCE_1);
+ ResourceUtils.initializeResourcesFromResourceInformationMap(riMap);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java
new file mode 100644
index 0000000..47d112d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestMagicCommitPaths.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.Lists;
+import org.junit.Assert;
+import org.junit.Test;
+
+import org.apache.hadoop.fs.Path;
+
+import static org.apache.hadoop.test.LambdaTestUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/**
+ * Tests for {@link MagicCommitPaths} path operations.
+ */
+public class TestMagicCommitPaths extends Assert {
+
+ private static final List<String> MAGIC_AT_ROOT =
+ list(MAGIC);
+ private static final List<String> MAGIC_AT_ROOT_WITH_CHILD =
+ list(MAGIC, "child");
+ private static final List<String> MAGIC_WITH_CHILD =
+ list("parent", MAGIC, "child");
+ private static final List<String> MAGIC_AT_WITHOUT_CHILD =
+ list("parent", MAGIC);
+
+ private static final List<String> DEEP_MAGIC =
+ list("parent1", "parent2", MAGIC, "child1", "child2");
+
+ public static final String[] EMPTY = {};
+
+ @Test
+ public void testSplitPathEmpty() throws Throwable {
+ intercept(IllegalArgumentException.class,
+ () -> splitPathToElements(new Path("")));
+ }
+
+ @Test
+ public void testSplitPathDoubleBackslash() {
+ assertPathSplits("//", EMPTY);
+ }
+
+ @Test
+ public void testSplitRootPath() {
+ assertPathSplits("/", EMPTY);
+ }
+
+ @Test
+ public void testSplitBasic() {
+ assertPathSplits("/a/b/c",
+ new String[]{"a", "b", "c"});
+ }
+
+ @Test
+ public void testSplitTrailingSlash() {
+ assertPathSplits("/a/b/c/",
+ new String[]{"a", "b", "c"});
+ }
+
+ @Test
+ public void testSplitShortPath() {
+ assertPathSplits("/a",
+ new String[]{"a"});
+ }
+
+ @Test
+ public void testSplitShortPathTrailingSlash() {
+ assertPathSplits("/a/",
+ new String[]{"a"});
+ }
+
+ @Test
+ public void testParentsMagicRoot() {
+ assertParents(EMPTY, MAGIC_AT_ROOT);
+ }
+
+ @Test
+ public void testChildrenMagicRoot() {
+ assertChildren(EMPTY, MAGIC_AT_ROOT);
+ }
+
+ @Test
+ public void testParentsMagicRootWithChild() {
+ assertParents(EMPTY, MAGIC_AT_ROOT_WITH_CHILD);
+ }
+
+ @Test
+ public void testChildMagicRootWithChild() {
+ assertChildren(a("child"), MAGIC_AT_ROOT_WITH_CHILD);
+ }
+
+ @Test
+ public void testChildrenMagicWithoutChild() {
+ assertChildren(EMPTY, MAGIC_AT_WITHOUT_CHILD);
+ }
+
+ @Test
+ public void testChildMagicWithChild() {
+ assertChildren(a("child"), MAGIC_WITH_CHILD);
+ }
+
+ @Test
+ public void testParentMagicWithChild() {
+ assertParents(a("parent"), MAGIC_WITH_CHILD);
+ }
+
+ @Test
+ public void testParentDeepMagic() {
+ assertParents(a("parent1", "parent2"), DEEP_MAGIC);
+ }
+
+ @Test
+ public void testChildrenDeepMagic() {
+ assertChildren(a("child1", "child2"), DEEP_MAGIC);
+ }
+
+ @Test
+ public void testLastElementEmpty() throws Throwable {
+ intercept(IllegalArgumentException.class,
+ () -> lastElement(new ArrayList<>(0)));
+ }
+
+ @Test
+ public void testLastElementSingle() {
+ assertEquals("first", lastElement(l("first")));
+ }
+
+ @Test
+ public void testLastElementDouble() {
+ assertEquals("2", lastElement(l("first", "2")));
+ }
+
+ @Test
+ public void testFinalDestinationNoMagic() {
+ assertEquals(l("first", "2"),
+ finalDestination(l("first", "2")));
+ }
+
+ @Test
+ public void testFinalDestinationMagic1() {
+ assertEquals(l("first", "2"),
+ finalDestination(l("first", MAGIC, "2")));
+ }
+
+ @Test
+ public void testFinalDestinationMagic2() {
+ assertEquals(l("first", "3.txt"),
+ finalDestination(l("first", MAGIC, "2", "3.txt")));
+ }
+
+ @Test
+ public void testFinalDestinationRootMagic2() {
+ assertEquals(l("3.txt"),
+ finalDestination(l(MAGIC, "2", "3.txt")));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFinalDestinationMagicNoChild() {
+ finalDestination(l(MAGIC));
+ }
+
+ @Test
+ public void testFinalDestinationBaseDirectChild() {
+ finalDestination(l(MAGIC, BASE, "3.txt"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testFinalDestinationBaseNoChild() {
+ assertEquals(l(), finalDestination(l(MAGIC, BASE)));
+ }
+
+ @Test
+ public void testFinalDestinationBaseSubdirsChild() {
+ assertEquals(l("2", "3.txt"),
+ finalDestination(l(MAGIC, "4", BASE, "2", "3.txt")));
+ }
+
+ /**
+ * If the base is above the magic dir, it's ignored.
+ */
+ @Test
+ public void testFinalDestinationIgnoresBaseBeforeMagic() {
+ assertEquals(l(BASE, "home", "3.txt"),
+ finalDestination(l(BASE, "home", MAGIC, "2", "3.txt")));
+ }
+
+ /** varargs to array. */
+ private static String[] a(String... str) {
+ return str;
+ }
+
+ /** list to array. */
+ private static List<String> l(String... str) {
+ return Arrays.asList(str);
+ }
+
+ /**
+ * Varags to list.
+ * @param args arguments
+ * @return a list
+ */
+ private static List<String> list(String... args) {
+ return Lists.newArrayList(args);
+ }
+
+ public void assertParents(String[] expected, List<String> elements) {
+ assertListEquals(expected, magicPathParents(elements));
+ }
+
+ public void assertChildren(String[] expected, List<String> elements) {
+ assertListEquals(expected, magicPathChildren(elements));
+ }
+
+ private void assertPathSplits(String pathString, String[] expected) {
+ Path path = new Path(pathString);
+ assertArrayEquals("From path " + path, expected,
+ splitPathToElements(path).toArray());
+ }
+
+ private void assertListEquals(String[] expected, List<String> actual) {
+ assertArrayEquals(expected, actual.toArray());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java
new file mode 100644
index 0000000..5e6fb82
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/TestTasks.java
@@ -0,0 +1,550 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test Tasks class.
+ */
+@RunWith(Parameterized.class)
+public class TestTasks extends HadoopTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(TestTasks.class);
+ public static final int ITEM_COUNT = 16;
+ private static final int FAILPOINT = 8;
+
+ private final int numThreads;
+ /**
+ * Thread pool for task execution.
+ */
+ private ExecutorService threadPool;
+ private final CounterTask failingTask
+ = new CounterTask("failing committer", FAILPOINT, Item::commit);
+
+ private final FailureCounter failures
+ = new FailureCounter("failures", 0, null);
+ private final CounterTask reverter
+ = new CounterTask("reverter", 0, Item::revert);
+ private final CounterTask aborter
+ = new CounterTask("aborter", 0, Item::abort);
+
+ /**
+ * Test array for parameterized test runs: how many threads and
+ * to use. Threading makes some of the assertions brittle; there are
+ * more checks on single thread than parallel ops.
+ * @return a list of parameter tuples.
+ */
+ @Parameterized.Parameters
+ public static Collection<Object[]> params() {
+ return Arrays.asList(new Object[][]{
+ {0},
+ {1},
+ {3},
+ {8},
+ {16},
+ });
+ }
+
+
+ private List<Item> items;
+
+
+ /**
+ * Construct the parameterized test.
+ * @param numThreads number of threads
+ */
+ public TestTasks(int numThreads) {
+ this.numThreads = numThreads;
+ }
+
+ /**
+ * In a parallel test run there is more than one thread doing the execution.
+ * @return true if the threadpool size is >1
+ */
+ public boolean isParallel() {
+ return numThreads > 1;
+ }
+
+ @Before
+ public void setup() {
+ items = IntStream.rangeClosed(1, ITEM_COUNT)
+ .mapToObj(i -> new Item(i,
+ String.format("With %d threads", numThreads)))
+ .collect(Collectors.toList());
+
+ if (numThreads > 0) {
+ threadPool = Executors.newFixedThreadPool(numThreads,
+ new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat(getMethodName() + "-pool-%d")
+ .build());
+ }
+
+ }
+
+ @After
+ public void teardown() {
+ if (threadPool != null) {
+ threadPool.shutdown();
+ threadPool = null;
+ }
+ }
+
+ /**
+ * create the builder.
+ * @return pre-inited builder
+ */
+ private Tasks.Builder<Item> builder() {
+ return Tasks.foreach(items).executeWith(threadPool);
+ }
+
+ private void assertRun(Tasks.Builder<Item> builder,
+ CounterTask task) throws IOException {
+ boolean b = builder.run(task);
+ assertTrue("Run of " + task + " failed", b);
+ }
+
+ private void assertFailed(Tasks.Builder<Item> builder,
+ CounterTask task) throws IOException {
+ boolean b = builder.run(task);
+ assertFalse("Run of " + task + " unexpectedly succeeded", b);
+ }
+
+ private String itemsToString() {
+ return "[" + items.stream().map(Item::toString)
+ .collect(Collectors.joining("\n")) +"]";
+ }
+
+ @Test
+ public void testSimpleInvocation() throws Throwable {
+ CounterTask t = new CounterTask("simple", 0, Item::commit);
+ assertRun(builder(), t);
+ t.assertInvoked("", ITEM_COUNT);
+ }
+
+ @Test
+ public void testFailNoStoppingSuppressed() throws Throwable {
+ assertFailed(builder().suppressExceptions(), failingTask);
+ failingTask.assertInvoked("Continued through operations", ITEM_COUNT);
+ items.forEach(Item::assertCommittedOrFailed);
+ }
+
+ @Test
+ public void testFailFastSuppressed() throws Throwable {
+ assertFailed(builder()
+ .suppressExceptions()
+ .stopOnFailure(),
+ failingTask);
+ if (isParallel()) {
+ failingTask.assertInvokedAtLeast("stop fast", FAILPOINT);
+ } else {
+ failingTask.assertInvoked("stop fast", FAILPOINT);
+ }
+ }
+
+ @Test
+ public void testFailedCallAbortSuppressed() throws Throwable {
+ assertFailed(builder()
+ .stopOnFailure()
+ .suppressExceptions()
+ .abortWith(aborter),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ if (!isParallel()) {
+ aborter.assertInvokedAtLeast("abort", 1);
+ // all uncommitted items were aborted
+ items.stream().filter(i -> !i.committed)
+ .map(Item::assertAborted);
+ items.stream().filter(i -> i.committed)
+ .forEach(i -> assertFalse(i.toString(), i.aborted));
+ }
+ }
+
+ @Test
+ public void testFailedCalledWhenNotStoppingSuppressed() throws Throwable {
+ assertFailed(builder()
+ .suppressExceptions()
+ .onFailure(failures),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+ @Test
+ public void testFailFastCallRevertSuppressed() throws Throwable {
+ assertFailed(builder()
+ .stopOnFailure()
+ .revertWith(reverter)
+ .abortWith(aborter)
+ .suppressExceptions()
+ .onFailure(failures),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ if (!isParallel()) {
+ aborter.assertInvokedAtLeast("abort", 1);
+ // all uncommitted items were aborted
+ items.stream().filter(i -> !i.committed)
+ .filter(i -> !i.failed)
+ .forEach(Item::assertAborted);
+ }
+ // all committed were reverted
+ items.stream().filter(i -> i.committed && !i.failed)
+ .forEach(Item::assertReverted);
+ // all reverted items are committed
+ items.stream().filter(i -> i.reverted)
+ .forEach(Item::assertCommitted);
+
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+ @Test
+ public void testFailSlowCallRevertSuppressed() throws Throwable {
+ assertFailed(builder()
+ .suppressExceptions()
+ .revertWith(reverter)
+ .onFailure(failures),
+ failingTask);
+ failingTask.assertInvokedAtLeast("success", FAILPOINT);
+ // all committed were reverted
+ // identify which task failed from the set
+ int failing = failures.getItem().id;
+ items.stream()
+ .filter(i -> i.id != failing)
+ .filter(i -> i.committed)
+ .forEach(Item::assertReverted);
+ // all reverted items are committed
+ items.stream().filter(i -> i.reverted)
+ .forEach(Item::assertCommitted);
+
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+ @Test
+ public void testFailFastExceptions() throws Throwable {
+ intercept(IOException.class,
+ () -> builder()
+ .stopOnFailure()
+ .run(failingTask));
+ if (isParallel()) {
+ failingTask.assertInvokedAtLeast("stop fast", FAILPOINT);
+ } else {
+ failingTask.assertInvoked("stop fast", FAILPOINT);
+ }
+ }
+
+ @Test
+ public void testFailSlowExceptions() throws Throwable {
+ intercept(IOException.class,
+ () -> builder()
+ .run(failingTask));
+ failingTask.assertInvoked("continued through operations", ITEM_COUNT);
+ items.forEach(Item::assertCommittedOrFailed);
+ }
+
+ @Test
+ public void testFailFastExceptionsWithAbortFailure() throws Throwable {
+ CounterTask failFirst = new CounterTask("task", 1, Item::commit);
+ CounterTask a = new CounterTask("aborter", 1, Item::abort);
+ intercept(IOException.class,
+ () -> builder()
+ .stopOnFailure()
+ .abortWith(a)
+ .run(failFirst));
+ if (!isParallel()) {
+ // expect the other tasks to be aborted
+ a.assertInvokedAtLeast("abort", ITEM_COUNT - 1);
+ }
+ }
+
+ @Test
+ public void testFailFastExceptionsWithAbortFailureStopped() throws Throwable {
+ CounterTask failFirst = new CounterTask("task", 1, Item::commit);
+ CounterTask a = new CounterTask("aborter", 1, Item::abort);
+ intercept(IOException.class,
+ () -> builder()
+ .stopOnFailure()
+ .stopAbortsOnFailure()
+ .abortWith(a)
+ .run(failFirst));
+ if (!isParallel()) {
+ // expect the other tasks to be aborted
+ a.assertInvoked("abort", 1);
+ }
+ }
+
+ /**
+ * Fail the last one committed, all the rest will be reverted.
+ * The actual ID of the last task has to be picke dup from the
+ * failure callback, as in the pool it may be one of any.
+ */
+ @Test
+ public void testRevertAllSuppressed() throws Throwable {
+ CounterTask failLast = new CounterTask("task", ITEM_COUNT, Item::commit);
+
+ assertFailed(builder()
+ .suppressExceptions()
+ .stopOnFailure()
+ .revertWith(reverter)
+ .abortWith(aborter)
+ .onFailure(failures),
+ failLast);
+ failLast.assertInvoked("success", ITEM_COUNT);
+ int abCount = aborter.getCount();
+ int revCount = reverter.getCount();
+ assertEquals(ITEM_COUNT, 1 + abCount + revCount);
+ // identify which task failed from the set
+ int failing = failures.getItem().id;
+ // all committed were reverted
+ items.stream()
+ .filter(i -> i.id != failing)
+ .filter(i -> i.committed)
+ .forEach(Item::assertReverted);
+ items.stream()
+ .filter(i -> i.id != failing)
+ .filter(i -> !i.committed)
+ .forEach(Item::assertAborted);
+ // all reverted items are committed
+ items.stream().filter(i -> i.reverted)
+ .forEach(Item::assertCommitted);
+
+ // only one failure was triggered
+ failures.assertInvoked("failure event", 1);
+ }
+
+
+ /**
+ * The Item which tasks process.
+ */
+ private final class Item {
+ private final int id;
+ private final String text;
+
+ private volatile boolean committed, aborted, reverted, failed;
+
+ private Item(int item, String text) {
+ this.id = item;
+ this.text = text;
+ }
+
+ boolean commit() {
+ committed = true;
+ return true;
+ }
+
+ boolean abort() {
+ aborted = true;
+ return true;
+ }
+
+ boolean revert() {
+ reverted = true;
+ return true;
+ }
+
+ boolean fail() {
+ failed = true;
+ return true;
+ }
+
+ public Item assertCommitted() {
+ assertTrue(toString() + " was not committed in\n"
+ + itemsToString(),
+ committed);
+ return this;
+ }
+
+ public Item assertCommittedOrFailed() {
+ assertTrue(toString() + " was not committed nor failed in\n"
+ + itemsToString(),
+ committed || failed);
+ return this;
+ }
+
+ public Item assertAborted() {
+ assertTrue(toString() + " was not aborted in\n"
+ + itemsToString(),
+ aborted);
+ return this;
+ }
+
+ public Item assertReverted() {
+ assertTrue(toString() + " was not reverted in\n"
+ + itemsToString(),
+ reverted);
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("Item{");
+ sb.append(String.format("[%02d]", id));
+ sb.append(", committed=").append(committed);
+ sb.append(", aborted=").append(aborted);
+ sb.append(", reverted=").append(reverted);
+ sb.append(", failed=").append(failed);
+ sb.append(", text=").append(text);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Class which can count invocations and, if limit > 0, will raise
+ * an exception on the specific invocation of {@link #note(Object)}
+ * whose count == limit.
+ */
+ private class BaseCounter {
+ private final AtomicInteger counter = new AtomicInteger(0);
+ private final int limit;
+ private final String name;
+ private Item item;
+ private final Optional<Function<Item, Boolean>> action;
+
+ /**
+ * Base counter, tracks items.
+ * @param name name for string/exception/logs.
+ * @param limit limit at which an exception is raised, 0 == never
+ * @param action optional action to invoke after the increment,
+ * before limit check
+ */
+ BaseCounter(String name,
+ int limit,
+ Function<Item, Boolean> action) {
+ this.name = name;
+ this.limit = limit;
+ this.action = Optional.ofNullable(action);
+ }
+
+ /**
+ * Apply the action to an item; log at info afterwards with both the
+ * before and after string values of the item.
+ * @param i item to process.
+ * @throws IOException failure in the action
+ */
+ void process(Item i) throws IOException {
+ this.item = i;
+ int count = counter.incrementAndGet();
+ if (limit == count) {
+ i.fail();
+ LOG.info("{}: Failed {}", this, i);
+ throw new IOException(String.format("%s: Limit %d reached for %s",
+ this, limit, i));
+ }
+ String before = i.toString();
+ action.map(a -> a.apply(i));
+ LOG.info("{}: {} -> {}", this, before, i);
+ }
+
+ int getCount() {
+ return counter.get();
+ }
+
+ Item getItem() {
+ return item;
+ }
+
+ void assertInvoked(String text, int expected) {
+ assertEquals(toString() + ": " + text, expected, getCount());
+ }
+
+ void assertInvokedAtLeast(String text, int expected) {
+ int actual = getCount();
+ assertTrue(toString() + ": " + text
+ + "-expected " + expected
+ + " invocations, but got " + actual
+ + " in " + itemsToString(),
+ expected <= actual);
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "BaseCounter{");
+ sb.append("name='").append(name).append('\'');
+ sb.append(", count=").append(counter.get());
+ sb.append(", limit=").append(limit);
+ sb.append(", item=").append(item);
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ private final class CounterTask
+ extends BaseCounter implements Tasks.Task<Item, IOException> {
+
+ private CounterTask(String name, int limit,
+ Function<Item, Boolean> action) {
+ super(name, limit, action);
+ }
+
+ @Override
+ public void run(Item item) throws IOException {
+ process(item);
+ }
+
+ }
+
+ private final class FailureCounter
+ extends BaseCounter implements Tasks.FailureTask<Item, IOException> {
+ private Exception exception;
+
+ private FailureCounter(String name, int limit,
+ Function<Item, Boolean> action) {
+ super(name, limit, action);
+ }
+
+ @Override
+ public void run(Item item, Exception ex) throws IOException {
+ process(item);
+ this.exception = ex;
+ }
+
+ private Exception getException() {
+ return exception;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITMagicCommitMRJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITMagicCommitMRJob.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITMagicCommitMRJob.java
new file mode 100644
index 0000000..57eb8b2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITMagicCommitMRJob.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.commit.AbstractITCommitMRJob;
+import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/**
+ * Full integration test for the Magic Committer.
+ *
+ * There's no need to disable the committer setting for the filesystem here,
+ * because the committers are being instantiated in their own processes;
+ * the settings in {@link #applyCustomConfigOptions(Configuration)} are
+ * passed down to these processes.
+ */
+public class ITMagicCommitMRJob extends AbstractITCommitMRJob {
+
+ /**
+ * Need consistency here.
+ * @return false
+ */
+ @Override
+ public boolean useInconsistentClient() {
+ return false;
+ }
+
+ @Override
+ protected String committerName() {
+ return MagicS3GuardCommitter.NAME;
+ }
+
+ /**
+ * Turn on the magic commit support for the FS, else nothing will work.
+ * @param conf configuration
+ */
+ @Override
+ protected void applyCustomConfigOptions(Configuration conf) {
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ }
+
+ /**
+ * Check that the magic dir was cleaned up.
+ * {@inheritDoc}
+ */
+ @Override
+ protected void customPostExecutionValidation(Path destPath,
+ SuccessData successData) throws Exception {
+ assertPathDoesNotExist("No cleanup", new Path(destPath, MAGIC));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java
new file mode 100644
index 0000000..74c1d9d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestMagicCommitProtocol.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.commit.AbstractITCommitProtocol;
+import org.apache.hadoop.fs.s3a.commit.AbstractS3ACommitter;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.CommitUtils;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjection;
+import org.apache.hadoop.fs.s3a.commit.CommitterFaultInjectionImpl;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobStatus;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/**
+ * Test the magic committer's commit protocol.
+ */
+public class ITestMagicCommitProtocol extends AbstractITCommitProtocol {
+
+ @Override
+ protected String suitename() {
+ return "ITestMagicCommitProtocol";
+ }
+
+ /**
+ * Need consistency here.
+ * @return false
+ */
+ @Override
+ public boolean useInconsistentClient() {
+ return false;
+ }
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration conf = super.createConfiguration();
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ return conf;
+ }
+
+ @Override
+ protected String getCommitterFactoryName() {
+ return CommitConstants.S3A_COMMITTER_FACTORY;
+ }
+
+ @Override
+ protected String getCommitterName() {
+ return CommitConstants.COMMITTER_NAME_MAGIC;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ CommitUtils.verifyIsMagicCommitFS(getFileSystem());
+ }
+
+ @Override
+ public void assertJobAbortCleanedUp(JobData jobData)
+ throws Exception {
+ // special handling of magic directory; harmless in staging
+ Path magicDir = new Path(getOutDir(), MAGIC);
+ ContractTestUtils.assertPathDoesNotExist(getFileSystem(),
+ "magic dir ", magicDir);
+ super.assertJobAbortCleanedUp(jobData);
+ }
+
+ @Override
+ protected AbstractS3ACommitter createCommitter(
+ Path outputPath,
+ TaskAttemptContext context)
+ throws IOException {
+ return new MagicS3GuardCommitter(outputPath, context);
+ }
+
+ public AbstractS3ACommitter createFailingCommitter(
+ TaskAttemptContext tContext) throws IOException {
+ return new CommitterWithFailedThenSucceed(getOutDir(), tContext);
+ }
+
+ protected void validateTaskAttemptPathDuringWrite(Path p) throws IOException {
+ String pathStr = p.toString();
+ assertTrue("not magic " + pathStr,
+ pathStr.contains(MAGIC));
+ assertPathDoesNotExist("task attempt visible", p);
+ }
+
+ protected void validateTaskAttemptPathAfterWrite(Path p) throws IOException {
+ FileStatus st = getFileSystem().getFileStatus(p);
+ assertEquals("file length in " + st, 0, st.getLen());
+ Path pendingFile = new Path(p.toString() + PENDING_SUFFIX);
+ assertPathExists("pending file", pendingFile);
+ }
+
+ /**
+ * The class provides a overridden implementation of commitJobInternal which
+ * causes the commit failed for the first time then succeed.
+ */
+
+ private static final class CommitterWithFailedThenSucceed extends
+ MagicS3GuardCommitter implements CommitterFaultInjection {
+ private final CommitterFaultInjectionImpl injection;
+
+ CommitterWithFailedThenSucceed(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ injection = new CommitterFaultInjectionImpl(outputPath, context, true);
+ }
+
+ @Override
+ public void setupJob(JobContext context) throws IOException {
+ injection.setupJob(context);
+ super.setupJob(context);
+ }
+
+ @Override
+ public void abortJob(JobContext context, JobStatus.State state)
+ throws IOException {
+ injection.abortJob(context, state);
+ super.abortJob(context, state);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void cleanupJob(JobContext context) throws IOException {
+ injection.cleanupJob(context);
+ super.cleanupJob(context);
+ }
+
+ @Override
+ public void setupTask(TaskAttemptContext context) throws IOException {
+ injection.setupTask(context);
+ super.setupTask(context);
+ }
+
+ @Override
+ public void commitTask(TaskAttemptContext context) throws IOException {
+ injection.commitTask(context);
+ super.commitTask(context);
+ }
+
+ @Override
+ public void abortTask(TaskAttemptContext context) throws IOException {
+ injection.abortTask(context);
+ super.abortTask(context);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ injection.commitJob(context);
+ super.commitJob(context);
+ }
+
+ @Override
+ public boolean needsTaskCommit(TaskAttemptContext context)
+ throws IOException {
+ injection.needsTaskCommit(context);
+ return super.needsTaskCommit(context);
+ }
+
+ @Override
+ public void setFaults(CommitterFaultInjection.Faults... faults) {
+ injection.setFaults(faults);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
new file mode 100644
index 0000000..e3a295b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/magic/ITestS3AHugeMagicCommits.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.magic;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.CommitOperations;
+import org.apache.hadoop.fs.s3a.commit.CommitUtils;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.scale.AbstractSTestS3AHugeFiles;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.*;
+
+
+/**
+ * Write a huge file via the magic commit mechanism,
+ * commit it and verify that it is there. This is needed to
+ * verify that the pending-upload mechanism works with multipart files
+ * of more than one part.
+ *
+ * This is a scale test.
+ */
+public class ITestS3AHugeMagicCommits extends AbstractSTestS3AHugeFiles {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ITestS3AHugeMagicCommits.class);
+
+ private Path magicDir;
+ private Path jobDir;
+
+ /** file used as the destination for the write;
+ * it is never actually created. */
+ private Path magicOutputFile;
+
+ /** The file with the JSON data about the commit. */
+ private Path pendingDataFile;
+
+ /**
+ * Use fast upload on disk.
+ * @return the upload buffer mechanism.
+ */
+ protected String getBlockOutputBufferName() {
+ return Constants.FAST_UPLOAD_BUFFER_DISK;
+ }
+
+ /**
+ * The suite name; required to be unique.
+ * @return the test suite name
+ */
+ @Override
+ public String getTestSuiteName() {
+ return "ITestS3AHugeMagicCommits";
+ }
+
+ /**
+ * Create the scale IO conf with the committer enabled.
+ * @return the configuration to use for the test FS.
+ */
+ @Override
+ protected Configuration createScaleConfiguration() {
+ Configuration conf = super.createScaleConfiguration();
+ conf.setBoolean(MAGIC_COMMITTER_ENABLED, true);
+ return conf;
+ }
+
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ CommitUtils.verifyIsMagicCommitFS(getFileSystem());
+
+ // set up the paths for the commit operation
+ Path finalDirectory = new Path(getScaleTestDir(), "commit");
+ magicDir = new Path(finalDirectory, MAGIC);
+ jobDir = new Path(magicDir, "job_001");
+ String filename = "commit.bin";
+ setHugefile(new Path(finalDirectory, filename));
+ magicOutputFile = new Path(jobDir, filename);
+ pendingDataFile = new Path(jobDir, filename + PENDING_SUFFIX);
+ }
+
+ /**
+ * Returns the path to the commit metadata file, not that of the huge file.
+ * @return a file in the job dir
+ */
+ @Override
+ protected Path getPathOfFileToCreate() {
+ return magicOutputFile;
+ }
+
+ @Override
+ public void test_030_postCreationAssertions() throws Throwable {
+ describe("Committing file");
+ assertPathDoesNotExist("final file exists", getHugefile());
+ assertPathExists("No pending file", pendingDataFile);
+ S3AFileSystem fs = getFileSystem();
+
+ // as a 0-byte marker is created, there is a file at the end path,
+ // it just MUST be 0-bytes long
+ FileStatus status = fs.getFileStatus(magicOutputFile);
+ assertEquals("Non empty marker file " + status,
+ 0, status.getLen());
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ CommitOperations operations = new CommitOperations(fs);
+ Path destDir = getHugefile().getParent();
+ assertPathExists("Magic dir", new Path(destDir, CommitConstants.MAGIC));
+ String destDirKey = fs.pathToKey(destDir);
+ List<String> uploads = listMultipartUploads(fs, destDirKey);
+
+ assertEquals("Pending uploads: "
+ + uploads.stream()
+ .collect(Collectors.joining("\n")), 1, uploads.size());
+ assertNotNull("jobDir", jobDir);
+ Pair<PendingSet, List<Pair<LocatedFileStatus, IOException>>>
+ results = operations.loadSinglePendingCommits(jobDir, false);
+ for (SinglePendingCommit singlePendingCommit :
+ results.getKey().getCommits()) {
+ operations.commitOrFail(singlePendingCommit);
+ }
+ timer.end("time to commit %s", pendingDataFile);
+ // upload is no longer pending
+ uploads = listMultipartUploads(fs, destDirKey);
+ assertEquals("Pending uploads"
+ + uploads.stream().collect(Collectors.joining("\n")),
+ 0, operations.listPendingUploadsUnderPath(destDir).size());
+ // at this point, the huge file exists, so the normal assertions
+ // on that file must be valid. Verify.
+ super.test_030_postCreationAssertions();
+ }
+
+ private void skipQuietly(String text) {
+ describe("Skipping: %s", text);
+ }
+
+ @Override
+ public void test_040_PositionedReadHugeFile() {
+ skipQuietly("test_040_PositionedReadHugeFile");
+ }
+
+ @Override
+ public void test_050_readHugeFile() {
+ skipQuietly("readHugeFile");
+ }
+
+ @Override
+ public void test_100_renameHugeFile() {
+ skipQuietly("renameHugeFile");
+ }
+
+ @Override
+ public void test_800_DeleteHugeFiles() throws IOException {
+ if (getFileSystem() != null) {
+ try {
+ getFileSystem().abortOutstandingMultipartUploads(0);
+ } catch (IOException e) {
+ LOG.info("Exception while purging old uploads", e);
+ }
+ }
+ try {
+ super.test_800_DeleteHugeFiles();
+ } finally {
+ ContractTestUtils.rm(getFileSystem(), magicDir, true, false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java
new file mode 100644
index 0000000..47383b7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/MockedStagingCommitter.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.ClientErrors;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingTestBase.ClientResults;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Committer subclass that uses a mocked S3A connection for testing.
+ */
+class MockedStagingCommitter extends StagingCommitter {
+
+ MockedStagingCommitter(Path outputPath,
+ TaskAttemptContext context)
+ throws IOException {
+ super(outputPath, context);
+ }
+
+ /**
+ * Returns the mock FS without checking FS type.
+ * @param out output path
+ * @param config job/task config
+ * @return a filesystem.
+ * @throws IOException IO failure
+ */
+ @Override
+ protected FileSystem getDestinationFS(Path out, Configuration config)
+ throws IOException {
+ return out.getFileSystem(config);
+ }
+
+ @Override
+ public void commitJob(JobContext context) throws IOException {
+ // turn off stamping an output marker, as that codepath isn't mocked yet.
+ super.commitJob(context);
+ Configuration conf = context.getConfiguration();
+ try {
+ String jobCommitterPath = conf.get("mock-results-file");
+ if (jobCommitterPath != null) {
+ try (ObjectOutputStream out = new ObjectOutputStream(
+ FileSystem.getLocal(conf)
+ .create(new Path(jobCommitterPath), false))) {
+ out.writeObject(getResults());
+ }
+ }
+ } catch (Exception e) {
+ // do nothing, the test will fail
+ }
+ }
+
+ @Override
+ protected void maybeCreateSuccessMarker(JobContext context,
+ List<String> filenames)
+ throws IOException {
+ //skipped
+ }
+
+ public ClientResults getResults() throws IOException {
+ MockS3AFileSystem mockFS = (MockS3AFileSystem)getDestS3AFS();
+ return mockFS.getOutcome().getKey();
+ }
+
+ public ClientErrors getErrors() throws IOException {
+ MockS3AFileSystem mockFS = (MockS3AFileSystem) getDestS3AFS();
+ return mockFS.getOutcome().getValue();
+ }
+
+ @Override
+ public String toString() {
+ return "MockedStagingCommitter{ " + super.toString() + " ";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedCommitterForTesting.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedCommitterForTesting.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedCommitterForTesting.java
new file mode 100644
index 0000000..5adce15
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/PartitionedCommitterForTesting.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+/**
+ * Partitioned committer overridden for better testing.
+ */
+class PartitionedCommitterForTesting extends
+ PartitionedStagingCommitter {
+
+ PartitionedCommitterForTesting(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ super(outputPath, context);
+ }
+
+ @Override
+ protected void initOutput(Path out) throws IOException {
+ super.initOutput(out);
+ setOutputPath(out);
+ }
+
+ /**
+ * Returns the mock FS without checking FS type.
+ * @param out output path
+ * @param config job/task config
+ * @return a filesystem.
+ * @throws IOException failure to get the FS
+ */
+ @Override
+ protected FileSystem getDestinationFS(Path out, Configuration config)
+ throws IOException {
+ return out.getFileSystem(config);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
new file mode 100644
index 0000000..38d5156
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/StagingTestBase.java
@@ -0,0 +1,724 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
+import com.amazonaws.services.s3.model.AmazonS3Exception;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
+import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
+import com.amazonaws.services.s3.model.DeleteObjectRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
+import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
+import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
+import com.amazonaws.services.s3.model.MultipartUpload;
+import com.amazonaws.services.s3.model.MultipartUploadListing;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.fs.s3a.MockS3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.AbstractCommitITest;
+import org.apache.hadoop.fs.s3a.commit.CommitConstants;
+import org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants;
+import org.apache.hadoop.fs.s3a.commit.MiniDFSClusterService;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.task.JobContextImpl;
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl;
+import org.apache.hadoop.mapreduce.v2.util.MRBuilderUtils;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+/**
+ * Test base for mock tests of staging committers:
+ * core constants and static methods, inner classes
+ * for specific test types.
+ *
+ * Some of the verification methods here are unused...they are being left
+ * in place in case changes on the implementation make the verifications
+ * relevant again.
+ */
+public class StagingTestBase {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(StagingTestBase.class);
+
+ public static final String BUCKET = MockS3AFileSystem.BUCKET;
+ public static final String OUTPUT_PREFIX = "output/path";
+ public static final Path OUTPUT_PATH =
+ new Path("s3a://" + BUCKET + "/" + OUTPUT_PREFIX);
+ public static final URI OUTPUT_PATH_URI = OUTPUT_PATH.toUri();
+
+ protected StagingTestBase() {
+ }
+
+ /**
+ * Sets up the mock filesystem instance and binds it to the
+ * {@link FileSystem#get(URI, Configuration)} call for the supplied URI
+ * and config.
+ * All standard mocking setup MUST go here.
+ * @param conf config to use
+ * @param outcome tuple of outcomes to store in mock FS
+ * @return the filesystem created
+ * @throws IOException IO problems.
+ */
+ protected static S3AFileSystem createAndBindMockFSInstance(Configuration conf,
+ Pair<StagingTestBase.ClientResults, StagingTestBase.ClientErrors> outcome)
+ throws IOException {
+ S3AFileSystem mockFs = mockS3AFileSystemRobustly();
+ MockS3AFileSystem wrapperFS = new MockS3AFileSystem(mockFs, outcome);
+ URI uri = OUTPUT_PATH_URI;
+ wrapperFS.initialize(uri, conf);
+ FileSystemTestHelper.addFileSystemForTesting(uri, conf, wrapperFS);
+ return mockFs;
+ }
+
+ private static S3AFileSystem mockS3AFileSystemRobustly() {
+ S3AFileSystem mockFS = mock(S3AFileSystem.class);
+ doNothing().when(mockFS).incrementReadOperations();
+ doNothing().when(mockFS).incrementWriteOperations();
+ doNothing().when(mockFS).incrementWriteOperations();
+ doNothing().when(mockFS).incrementWriteOperations();
+ return mockFS;
+ }
+
+ /**
+ * Look up the FS by URI, return a (cast) Mock wrapper.
+ * @param conf config
+ * @return the FS
+ * @throws IOException IO Failure
+ */
+ public static MockS3AFileSystem lookupWrapperFS(Configuration conf)
+ throws IOException {
+ return (MockS3AFileSystem) FileSystem.get(OUTPUT_PATH_URI, conf);
+ }
+
+ public static void verifyCompletion(FileSystem mockS3) throws IOException {
+ verifyCleanupTempFiles(mockS3);
+ verifyNoMoreInteractions(mockS3);
+ }
+
+ public static void verifyDeleted(FileSystem mockS3, Path path)
+ throws IOException {
+ verify(mockS3).delete(path, true);
+ }
+
+ public static void verifyDeleted(FileSystem mockS3, String child)
+ throws IOException {
+ verifyDeleted(mockS3, new Path(OUTPUT_PATH, child));
+ }
+
+ public static void verifyCleanupTempFiles(FileSystem mockS3)
+ throws IOException {
+ verifyDeleted(mockS3,
+ new Path(OUTPUT_PATH, CommitConstants.TEMPORARY));
+ }
+
+ protected static void assertConflictResolution(
+ StagingCommitter committer,
+ JobContext job,
+ ConflictResolution mode) {
+ Assert.assertEquals("Conflict resolution mode in " + committer,
+ mode, committer.getConflictResolutionMode(job, new Configuration()));
+ }
+
+ public static void pathsExist(FileSystem mockS3, String... children)
+ throws IOException {
+ for (String child : children) {
+ pathExists(mockS3, new Path(OUTPUT_PATH, child));
+ }
+ }
+
+ public static void pathExists(FileSystem mockS3, Path path)
+ throws IOException {
+ when(mockS3.exists(path)).thenReturn(true);
+ }
+
+ public static void pathDoesNotExist(FileSystem mockS3, Path path)
+ throws IOException {
+ when(mockS3.exists(path)).thenReturn(false);
+ }
+
+ public static void canDelete(FileSystem mockS3, String... children)
+ throws IOException {
+ for (String child : children) {
+ canDelete(mockS3, new Path(OUTPUT_PATH, child));
+ }
+ }
+
+ public static void canDelete(FileSystem mockS3, Path f) throws IOException {
+ when(mockS3.delete(f,
+ true /* recursive */))
+ .thenReturn(true);
+ }
+
+ public static void verifyExistenceChecked(FileSystem mockS3, String child)
+ throws IOException {
+ verifyExistenceChecked(mockS3, new Path(OUTPUT_PATH, child));
+ }
+
+ public static void verifyExistenceChecked(FileSystem mockS3, Path path)
+ throws IOException {
+ verify(mockS3).exists(path);
+ }
+
+ /**
+ * Provides setup/teardown of a MiniDFSCluster for tests that need one.
+ */
+ public static class MiniDFSTest extends HadoopTestBase {
+
+ private static MiniDFSClusterService hdfs;
+
+ private static JobConf conf = null;
+
+ protected static JobConf getConfiguration() {
+ return conf;
+ }
+
+ protected static FileSystem getDFS() {
+ return hdfs.getClusterFS();
+ }
+
+ /**
+ * Setup the mini HDFS cluster.
+ * @throws IOException Failure
+ */
+ @BeforeClass
+ @SuppressWarnings("deprecation")
+ public static void setupHDFS() throws IOException {
+ if (hdfs == null) {
+ JobConf c = new JobConf();
+ hdfs = new MiniDFSClusterService();
+ hdfs.init(c);
+ hdfs.start();
+ conf = c;
+ }
+ }
+
+ @SuppressWarnings("ThrowableNotThrown")
+ @AfterClass
+ public static void teardownFS() throws IOException {
+ ServiceOperations.stopQuietly(hdfs);
+ conf = null;
+ hdfs = null;
+ }
+
+ }
+
+ /**
+ * Base class for job committer tests.
+ * @param <C> committer
+ */
+ public abstract static class JobCommitterTest<C extends OutputCommitter>
+ extends HadoopTestBase {
+ private static final JobID JOB_ID = new JobID("job", 1);
+ private JobConf jobConf;
+
+ // created in BeforeClass
+ private S3AFileSystem mockFS = null;
+ private MockS3AFileSystem wrapperFS = null;
+ private JobContext job = null;
+
+ // created in Before
+ private StagingTestBase.ClientResults results = null;
+ private StagingTestBase.ClientErrors errors = null;
+ private AmazonS3 mockClient = null;
+
+ @Before
+ public void setupJob() throws Exception {
+ this.jobConf = new JobConf();
+ jobConf.set(InternalCommitterConstants.FS_S3A_COMMITTER_STAGING_UUID,
+ UUID.randomUUID().toString());
+ jobConf.setBoolean(
+ CommitConstants.CREATE_SUCCESSFUL_JOB_OUTPUT_DIR_MARKER,
+ false);
+
+ this.job = new JobContextImpl(jobConf, JOB_ID);
+ this.results = new StagingTestBase.ClientResults();
+ this.errors = new StagingTestBase.ClientErrors();
+ this.mockClient = newMockS3Client(results, errors);
+ this.mockFS = createAndBindMockFSInstance(jobConf,
+ Pair.of(results, errors));
+ this.wrapperFS = lookupWrapperFS(jobConf);
+ // and bind the FS
+ wrapperFS.setAmazonS3Client(mockClient);
+ }
+
+ public S3AFileSystem getMockS3A() {
+ return mockFS;
+ }
+
+ public MockS3AFileSystem getWrapperFS() {
+ return wrapperFS;
+ }
+
+ public JobContext getJob() {
+ return job;
+ }
+
+ /**
+ * Create a task attempt for a job by creating a stub task ID.
+ * @return a task attempt
+ */
+ public TaskAttemptContext createTaskAttemptForJob() {
+ return AbstractCommitITest.taskAttemptForJob(
+ MRBuilderUtils.newJobId(1, JOB_ID.getId(), 1), job);
+ }
+
+ protected StagingTestBase.ClientResults getMockResults() {
+ return results;
+ }
+
+ protected StagingTestBase.ClientErrors getMockErrors() {
+ return errors;
+ }
+
+ abstract C newJobCommitter() throws Exception;
+ }
+
+ /** Abstract test of task commits. */
+ public abstract static class TaskCommitterTest<C extends OutputCommitter>
+ extends JobCommitterTest<C> {
+ private static final TaskAttemptID AID = new TaskAttemptID(
+ new TaskID(JobCommitterTest.JOB_ID, TaskType.REDUCE, 2), 3);
+
+ private C jobCommitter = null;
+ private TaskAttemptContext tac = null;
+ private File tempDir;
+
+ @Before
+ public void setupTask() throws Exception {
+ this.jobCommitter = newJobCommitter();
+ jobCommitter.setupJob(getJob());
+
+ this.tac = new TaskAttemptContextImpl(
+ new Configuration(getJob().getConfiguration()), AID);
+
+ // get the task's configuration copy so modifications take effect
+ String tmp = System.getProperty(
+ StagingCommitterConstants.JAVA_IO_TMPDIR);
+ tempDir = new File(tmp);
+ tac.getConfiguration().set(Constants.BUFFER_DIR, tmp + "/buffer");
+ tac.getConfiguration().set(
+ CommitConstants.FS_S3A_COMMITTER_STAGING_TMP_PATH,
+ tmp + "/cluster");
+ }
+
+ protected C getJobCommitter() {
+ return jobCommitter;
+ }
+
+ protected TaskAttemptContext getTAC() {
+ return tac;
+ }
+
+ abstract C newTaskCommitter() throws Exception;
+
+ protected File getTempDir() {
+ return tempDir;
+ }
+ }
+
+ /**
+ * Results accrued during mock runs.
+ * This data is serialized in MR Tests and read back in in the test runner
+ */
+ public static class ClientResults implements Serializable {
+ private static final long serialVersionUID = -3137637327090709905L;
+ // For inspection of what the committer did
+ private final Map<String, InitiateMultipartUploadRequest> requests =
+ Maps.newHashMap();
+ private final List<String> uploads = Lists.newArrayList();
+ private final List<UploadPartRequest> parts = Lists.newArrayList();
+ private final Map<String, List<String>> tagsByUpload = Maps.newHashMap();
+ private final List<CompleteMultipartUploadRequest> commits =
+ Lists.newArrayList();
+ private final List<AbortMultipartUploadRequest> aborts
+ = Lists.newArrayList();
+ private final Map<String, String> activeUploads =
+ Maps.newHashMap();
+ private final List<DeleteObjectRequest> deletes = Lists.newArrayList();
+
+ public Map<String, InitiateMultipartUploadRequest> getRequests() {
+ return requests;
+ }
+
+ public List<String> getUploads() {
+ return uploads;
+ }
+
+ public List<UploadPartRequest> getParts() {
+ return parts;
+ }
+
+ public Map<String, List<String>> getTagsByUpload() {
+ return tagsByUpload;
+ }
+
+ public List<CompleteMultipartUploadRequest> getCommits() {
+ return commits;
+ }
+
+ public List<AbortMultipartUploadRequest> getAborts() {
+ return aborts;
+ }
+
+ public List<DeleteObjectRequest> getDeletes() {
+ return deletes;
+ }
+
+ public void resetDeletes() {
+ deletes.clear();
+ }
+
+ public void resetUploads() {
+ uploads.clear();
+ activeUploads.clear();
+ }
+
+ public void resetCommits() {
+ commits.clear();
+ }
+
+ public void resetRequests() {
+ requests.clear();
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ super.toString());
+ sb.append("{ requests=").append(requests.size());
+ sb.append(", uploads=").append(uploads.size());
+ sb.append(", parts=").append(parts.size());
+ sb.append(", tagsByUpload=").append(tagsByUpload.size());
+ sb.append(", commits=").append(commits.size());
+ sb.append(", aborts=").append(aborts.size());
+ sb.append(", deletes=").append(deletes.size());
+ sb.append('}');
+ return sb.toString();
+ }
+ }
+
+ /** Control errors to raise in mock S3 client. */
+ public static class ClientErrors {
+ // For injecting errors
+ private int failOnInit = -1;
+ private int failOnUpload = -1;
+ private int failOnCommit = -1;
+ private int failOnAbort = -1;
+ private boolean recover = false;
+
+ public void failOnInit(int initNum) {
+ this.failOnInit = initNum;
+ }
+
+ public void failOnUpload(int uploadNum) {
+ this.failOnUpload = uploadNum;
+ }
+
+ public void failOnCommit(int commitNum) {
+ this.failOnCommit = commitNum;
+ }
+
+ public void failOnAbort(int abortNum) {
+ this.failOnAbort = abortNum;
+ }
+
+ public void recoverAfterFailure() {
+ this.recover = true;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "ClientErrors{");
+ sb.append("failOnInit=").append(failOnInit);
+ sb.append(", failOnUpload=").append(failOnUpload);
+ sb.append(", failOnCommit=").append(failOnCommit);
+ sb.append(", failOnAbort=").append(failOnAbort);
+ sb.append(", recover=").append(recover);
+ sb.append('}');
+ return sb.toString();
+ }
+
+ public int getFailOnInit() {
+ return failOnInit;
+ }
+
+ public int getFailOnUpload() {
+ return failOnUpload;
+ }
+
+ public int getFailOnCommit() {
+ return failOnCommit;
+ }
+
+ public int getFailOnAbort() {
+ return failOnAbort;
+ }
+
+ public boolean isRecover() {
+ return recover;
+ }
+ }
+
+ /**
+ * Instantiate mock client with the results and errors requested.
+ * @param results results to accrue
+ * @param errors when (if any) to fail
+ * @return the mock client to patch in to a committer/FS instance
+ */
+ public static AmazonS3 newMockS3Client(final ClientResults results,
+ final ClientErrors errors) {
+ AmazonS3Client mockClient = mock(AmazonS3Client.class);
+ final Object lock = new Object();
+
+ // initiateMultipartUpload
+ when(mockClient
+ .initiateMultipartUpload(any(InitiateMultipartUploadRequest.class)))
+ .thenAnswer(invocation -> {
+ LOG.debug("initiateMultipartUpload for {}", mockClient);
+ synchronized (lock) {
+ if (results.requests.size() == errors.failOnInit) {
+ if (errors.recover) {
+ errors.failOnInit(-1);
+ }
+ throw new AmazonClientException(
+ "Mock Fail on init " + results.requests.size());
+ }
+ String uploadId = UUID.randomUUID().toString();
+ InitiateMultipartUploadRequest req = invocation.getArgumentAt(
+ 0, InitiateMultipartUploadRequest.class);
+ results.requests.put(uploadId, req);
+ results.activeUploads.put(uploadId, req.getKey());
+ results.uploads.add(uploadId);
+ return newResult(results.requests.get(uploadId), uploadId);
+ }
+ });
+
+ // uploadPart
+ when(mockClient.uploadPart(any(UploadPartRequest.class)))
+ .thenAnswer(invocation -> {
+ LOG.debug("uploadPart for {}", mockClient);
+ synchronized (lock) {
+ if (results.parts.size() == errors.failOnUpload) {
+ if (errors.recover) {
+ errors.failOnUpload(-1);
+ }
+ LOG.info("Triggering upload failure");
+ throw new AmazonClientException(
+ "Mock Fail on upload " + results.parts.size());
+ }
+ UploadPartRequest req = invocation.getArgumentAt(
+ 0, UploadPartRequest.class);
+ results.parts.add(req);
+ String etag = UUID.randomUUID().toString();
+ List<String> etags = results.tagsByUpload.get(req.getUploadId());
+ if (etags == null) {
+ etags = Lists.newArrayList();
+ results.tagsByUpload.put(req.getUploadId(), etags);
+ }
+ etags.add(etag);
+ return newResult(req, etag);
+ }
+ });
+
+ // completeMultipartUpload
+ when(mockClient
+ .completeMultipartUpload(any(CompleteMultipartUploadRequest.class)))
+ .thenAnswer(invocation -> {
+ LOG.debug("completeMultipartUpload for {}", mockClient);
+ synchronized (lock) {
+ if (results.commits.size() == errors.failOnCommit) {
+ if (errors.recover) {
+ errors.failOnCommit(-1);
+ }
+ throw new AmazonClientException(
+ "Mock Fail on commit " + results.commits.size());
+ }
+ CompleteMultipartUploadRequest req = invocation.getArgumentAt(
+ 0, CompleteMultipartUploadRequest.class);
+ results.commits.add(req);
+ results.activeUploads.remove(req.getUploadId());
+
+ return newResult(req);
+ }
+ });
+
+ // abortMultipartUpload mocking
+ doAnswer(invocation -> {
+ LOG.debug("abortMultipartUpload for {}", mockClient);
+ synchronized (lock) {
+ if (results.aborts.size() == errors.failOnAbort) {
+ if (errors.recover) {
+ errors.failOnAbort(-1);
+ }
+ throw new AmazonClientException(
+ "Mock Fail on abort " + results.aborts.size());
+ }
+ AbortMultipartUploadRequest req = invocation.getArgumentAt(
+ 0, AbortMultipartUploadRequest.class);
+ String id = req.getUploadId();
+ String p = results.activeUploads.remove(id);
+ if (p == null) {
+ // upload doesn't exist
+ AmazonS3Exception ex = new AmazonS3Exception(
+ "not found " + id);
+ ex.setStatusCode(404);
+ throw ex;
+ }
+ results.aborts.add(req);
+ return null;
+ }
+ })
+ .when(mockClient)
+ .abortMultipartUpload(any(AbortMultipartUploadRequest.class));
+
+ // deleteObject mocking
+ doAnswer(invocation -> {
+ LOG.debug("deleteObject for {}", mockClient);
+ synchronized (lock) {
+ results.deletes.add(invocation.getArgumentAt(
+ 0, DeleteObjectRequest.class));
+ return null;
+ }
+ })
+ .when(mockClient)
+ .deleteObject(any(DeleteObjectRequest.class));
+
+ // deleteObject mocking
+ doAnswer(invocation -> {
+ LOG.debug("deleteObject for {}", mockClient);
+ synchronized (lock) {
+ results.deletes.add(new DeleteObjectRequest(
+ invocation.getArgumentAt(0, String.class),
+ invocation.getArgumentAt(1, String.class)
+ ));
+ return null;
+ }
+ }).when(mockClient)
+ .deleteObject(any(String.class), any(String.class));
+
+ // to String returns the debug information
+ when(mockClient.toString()).thenAnswer(
+ invocation -> "Mock3AClient " + results + " " + errors);
+
+ when(mockClient
+ .listMultipartUploads(any(ListMultipartUploadsRequest.class)))
+ .thenAnswer(invocation -> {
+ synchronized (lock) {
+ MultipartUploadListing l = new MultipartUploadListing();
+ l.setMultipartUploads(
+ results.activeUploads.entrySet().stream()
+ .map(e -> newMPU(e.getKey(), e.getValue()))
+ .collect(Collectors.toList()));
+ return l;
+ }
+ });
+
+ return mockClient;
+ }
+
+ private static CompleteMultipartUploadResult newResult(
+ CompleteMultipartUploadRequest req) {
+ return new CompleteMultipartUploadResult();
+ }
+
+
+ private static MultipartUpload newMPU(String id, String path) {
+ MultipartUpload up = new MultipartUpload();
+ up.setUploadId(id);
+ up.setKey(path);
+ return up;
+ }
+
+ private static UploadPartResult newResult(UploadPartRequest request,
+ String etag) {
+ UploadPartResult result = new UploadPartResult();
+ result.setPartNumber(request.getPartNumber());
+ result.setETag(etag);
+ return result;
+ }
+
+ private static InitiateMultipartUploadResult newResult(
+ InitiateMultipartUploadRequest request, String uploadId) {
+ InitiateMultipartUploadResult result = new InitiateMultipartUploadResult();
+ result.setUploadId(uploadId);
+ return result;
+ }
+
+ /**
+ * create files in the attempt path that should be found by
+ * {@code getTaskOutput}.
+ * @param relativeFiles list of files relative to address path
+ * @param attemptPath attempt path
+ * @param conf config for FS
+ * @throws IOException on any failure
+ */
+ public static void createTestOutputFiles(List<String> relativeFiles,
+ Path attemptPath,
+ Configuration conf) throws IOException {
+ //
+ FileSystem attemptFS = attemptPath.getFileSystem(conf);
+ attemptFS.delete(attemptPath, true);
+ for (String relative : relativeFiles) {
+ // 0-length files are ignored, so write at least one byte
+ OutputStream out = attemptFS.create(new Path(attemptPath, relative));
+ out.write(34);
+ out.close();
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestPaths.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestPaths.java
new file mode 100644
index 0000000..ee6480a
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestPaths.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit.staging;
+
+import org.junit.Test;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.HadoopTestBase;
+
+import static org.apache.hadoop.fs.s3a.commit.staging.Paths.*;
+import static org.apache.hadoop.test.LambdaTestUtils.intercept;
+
+/**
+ * Test {@link org.apache.hadoop.fs.s3a.commit.staging.Paths}.
+ */
+public class TestPaths extends HadoopTestBase {
+
+ @Test
+ public void testUUIDPart() {
+ assertUUIDAdded("/part-0000", "/part-0000-UUID");
+ }
+
+ @Test
+ public void testUUIDPartSuffix() {
+ assertUUIDAdded("/part-0000.gz.csv", "/part-0000-UUID.gz.csv");
+ }
+
+ @Test
+ public void testUUIDDottedPath() {
+ assertUUIDAdded("/parent.dir/part-0000", "/parent.dir/part-0000-UUID");
+ }
+
+ @Test
+ public void testUUIDPartUUID() {
+ assertUUIDAdded("/part-0000-UUID.gz.csv", "/part-0000-UUID.gz.csv");
+ }
+
+ @Test
+ public void testUUIDParentUUID() {
+ assertUUIDAdded("/UUID/part-0000.gz.csv", "/UUID/part-0000.gz.csv");
+ }
+
+ @Test
+ public void testUUIDDir() throws Throwable {
+ intercept(IllegalStateException.class,
+ () -> addUUID("/dest/", "UUID"));
+ }
+
+ @Test
+ public void testUUIDEmptyDir() throws Throwable {
+ intercept(IllegalArgumentException.class,
+ () -> addUUID("", "UUID"));
+ }
+
+ @Test
+ public void testEmptyUUID() throws Throwable {
+ intercept(IllegalArgumentException.class,
+ () -> addUUID("part-0000.gz", ""));
+ }
+
+ private void assertUUIDAdded(String path, String expected) {
+ assertEquals("from " + path, expected, addUUID(path, "UUID"));
+ }
+
+ private static final String DATA = "s3a://landsat-pds/data/";
+ private static final Path BASE = new Path(DATA);
+
+ @Test
+ public void testRelativizeOneLevel() {
+ String suffix = "year=2017";
+ Path path = new Path(DATA + suffix);
+ assertEquals(suffix, getRelativePath(BASE, path));
+ }
+
+ @Test
+ public void testRelativizeTwoLevel() {
+ String suffix = "year=2017/month=10";
+ Path path = path(BASE, suffix);
+ assertEquals(suffix, getRelativePath(BASE, path));
+ }
+
+ @Test
+ public void testRelativizeSelf() {
+ assertEquals("", getRelativePath(BASE, BASE));
+ }
+
+ @Test
+ public void testRelativizeParent() {
+ // goes up to the parent if one is above the other
+ assertEquals("/", getRelativePath(BASE, BASE.getParent()));
+ }
+
+ @Test
+ public void testGetPartition() {
+ assertEquals("year=2017/month=10",
+ getPartition("year=2017/month=10/part-0000.avro"));
+ }
+
+ @Test
+ public void testMPUCommitDir() throws Throwable {
+ Configuration conf = new Configuration();
+ LocalFileSystem localFS = FileSystem.getLocal(conf);
+ Path dir = getMultipartUploadCommitsDirectory(localFS, conf, "UUID");
+ assertTrue(dir.toString().endsWith("UUID/"
+ + StagingCommitterConstants.STAGING_UPLOADS));
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/49] hadoop git commit: YARN-7509. AsyncScheduleThread and
ResourceCommitterService are still running after RM is transitioned to
standby. (Tao Yang via wangda)
Posted by su...@apache.org.
YARN-7509. AsyncScheduleThread and ResourceCommitterService are still running after RM is transitioned to standby. (Tao Yang via wangda)
Change-Id: I7477fe355419fd4a0a6e2bdda7319abad4c4c748
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/834e91ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/834e91ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/834e91ee
Branch: refs/heads/YARN-5881
Commit: 834e91ee91d22d74866afbf6252107e969bf8370
Parents: d162252
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 23 19:59:03 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Nov 23 19:59:03 2017 -0800
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 16 +-
.../TestRMHAForAsyncScheduler.java | 155 +++++++++++++++++++
2 files changed, 164 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/834e91ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index ed30ad1..218adf3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -459,7 +459,7 @@ public class CapacityScheduler extends
* Schedule on all nodes by starting at a random point.
* @param cs
*/
- static void schedule(CapacityScheduler cs) {
+ static void schedule(CapacityScheduler cs) throws InterruptedException{
// First randomize the start point
int current = 0;
Collection<FiCaSchedulerNode> nodes = cs.nodeTracker.getAllNodes();
@@ -475,9 +475,7 @@ public class CapacityScheduler extends
cs.allocateContainersToNode(node.getNodeID(), false);
}
- try {
- Thread.sleep(cs.getAsyncScheduleInterval());
- } catch (InterruptedException e) {}
+ Thread.sleep(cs.getAsyncScheduleInterval());
}
static class AsyncScheduleThread extends Thread {
@@ -492,9 +490,9 @@ public class CapacityScheduler extends
@Override
public void run() {
- while (true) {
+ while (!Thread.currentThread().isInterrupted()) {
try {
- if (!runSchedules.get() || Thread.currentThread().isInterrupted()) {
+ if (!runSchedules.get()) {
Thread.sleep(100);
} else {
// Don't run schedule if we have some pending backlogs already
@@ -505,9 +503,11 @@ public class CapacityScheduler extends
}
}
} catch (InterruptedException ie) {
- // Do nothing
+ // keep interrupt signal
+ Thread.currentThread().interrupt();
}
}
+ LOG.info("AsyncScheduleThread[" + getName() + "] exited!");
}
public void beginSchedule() {
@@ -546,8 +546,10 @@ public class CapacityScheduler extends
} catch (InterruptedException e) {
LOG.error(e);
+ Thread.currentThread().interrupt();
}
}
+ LOG.info("ResourceCommitterService exited!");
}
public void addNewCommitRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/834e91ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java
new file mode 100644
index 0000000..46d5cda
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForAsyncScheduler.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.ha.HAServiceProtocol;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestRMHAForAsyncScheduler extends RMHATestBase {
+
+ @Before
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+ confForRM1
+ .setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+ confForRM1.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ confForRM1.setBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true);
+
+ confForRM2
+ .setClass(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class, ResourceCalculator.class);
+ confForRM2.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ confForRM2.setBoolean(
+ CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_ENABLE, true);
+ }
+
+ @Test(timeout = 60000)
+ public void testAsyncScheduleThreadStateAfterRMHATransit() throws Exception {
+ // start two RMs, and transit rm1 to active, rm2 to standby
+ startRMs();
+ // register NM
+ rm1.registerNode("h1:1234", 8192, 8);
+ // submit app1 and check
+ RMApp app1 = submitAppAndCheckLaunched(rm1);
+
+ // failover RM1 to RM2
+ explicitFailover();
+ checkAsyncSchedulerThreads(Thread.currentThread());
+
+ // register NM, kill app1
+ rm2.registerNode("h1:1234", 8192, 8);
+ rm2.waitForState(app1.getCurrentAppAttempt().getAppAttemptId(),
+ RMAppAttemptState.LAUNCHED);
+ rm2.killApp(app1.getApplicationId());
+ // submit app3 and check
+ RMApp app2 = submitAppAndCheckLaunched(rm2);
+
+ // failover RM2 to RM1
+ HAServiceProtocol.StateChangeRequestInfo requestInfo =
+ new HAServiceProtocol.StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ rm2.adminService.transitionToStandby(requestInfo);
+ rm1.adminService.transitionToActive(requestInfo);
+ Assert.assertTrue(rm2.getRMContext().getHAServiceState()
+ == HAServiceProtocol.HAServiceState.STANDBY);
+ Assert.assertTrue(rm1.getRMContext().getHAServiceState()
+ == HAServiceProtocol.HAServiceState.ACTIVE);
+ // check async schedule threads
+ checkAsyncSchedulerThreads(Thread.currentThread());
+
+ // register NM, kill app2
+ rm1.registerNode("h1:1234", 8192, 8);
+ rm1.waitForState(app2.getCurrentAppAttempt().getAppAttemptId(),
+ RMAppAttemptState.LAUNCHED);
+ rm1.killApp(app2.getApplicationId());
+ // submit app3 and check
+ submitAppAndCheckLaunched(rm1);
+
+ rm1.stop();
+ rm2.stop();
+ }
+
+ private RMApp submitAppAndCheckLaunched(MockRM rm) throws Exception {
+ RMApp app = rm.submitApp(200, "",
+ UserGroupInformation.getCurrentUser().getShortUserName(), null, false,
+ "default", configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, false,
+ false);
+ rm.waitForState(app.getApplicationId(), RMAppState.ACCEPTED);
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ rm.sendAMLaunched(attempt.getAppAttemptId());
+ rm.waitForState(app.getCurrentAppAttempt().getAppAttemptId(),
+ RMAppAttemptState.LAUNCHED);
+ return app;
+ }
+
+ /**
+ * Make sure the state of async-scheduler threads is correct
+ * @param currentThread
+ */
+ private void checkAsyncSchedulerThreads(Thread currentThread){
+ // Make sure AsyncScheduleThread is interrupted
+ ThreadGroup threadGroup = currentThread.getThreadGroup();
+ while (threadGroup.getParent() != null) {
+ threadGroup = threadGroup.getParent();
+ }
+ Thread[] threads = new Thread[threadGroup.activeCount()];
+ threadGroup.enumerate(threads);
+ int numAsyncScheduleThread = 0;
+ int numResourceCommitterService = 0;
+ Thread asyncScheduleThread = null;
+ Thread resourceCommitterService = null;
+ for (Thread thread : threads) {
+ StackTraceElement[] stackTrace = thread.getStackTrace();
+ if(stackTrace.length>0){
+ String stackBottom = stackTrace[stackTrace.length-1].toString();
+ if(stackBottom.contains("AsyncScheduleThread.run")){
+ numAsyncScheduleThread++;
+ asyncScheduleThread = thread;
+ }else if(stackBottom.contains("ResourceCommitterService.run")){
+ numResourceCommitterService++;
+ resourceCommitterService = thread;
+ }
+ }
+ }
+ Assert.assertEquals(1, numResourceCommitterService);
+ Assert.assertEquals(1, numAsyncScheduleThread);
+ Assert.assertNotNull(asyncScheduleThread);
+ Assert.assertNotNull(resourceCommitterService);
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/49] hadoop git commit: YARN-6483. Add nodes transitioning to
DECOMMISSIONING state to the list of updated nodes returned to the AM. (Juan
Rodriguez Hortala via asuresh)
Posted by su...@apache.org.
YARN-6483. Add nodes transitioning to DECOMMISSIONING state to the list of updated nodes returned to the AM. (Juan Rodriguez Hortala via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b46ca7e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b46ca7e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b46ca7e7
Branch: refs/heads/YARN-5881
Commit: b46ca7e73b8bac3fdbff0b13afe009308078acf2
Parents: aab4395
Author: Arun Suresh <as...@apache.org>
Authored: Wed Nov 22 19:16:44 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Wed Nov 22 19:18:30 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/NodeReport.java | 47 ++++++--
.../hadoop/yarn/api/records/NodeUpdateType.java | 29 +++++
.../src/main/proto/yarn_protos.proto | 8 ++
.../hadoop/yarn/client/ProtocolHATestBase.java | 14 +--
.../hadoop/yarn/client/cli/TestYarnCLI.java | 2 +-
.../api/records/impl/pb/NodeReportPBImpl.java | 50 +++++++-
.../yarn/api/records/impl/pb/ProtoUtils.java | 12 ++
.../hadoop/yarn/server/utils/BuilderUtils.java | 14 ++-
.../server/resourcemanager/ClientRMService.java | 5 +-
.../DecommissioningNodesWatcher.java | 38 +-----
.../resourcemanager/DefaultAMSProcessor.java | 12 +-
.../resourcemanager/NodesListManager.java | 78 +++++++++----
.../NodesListManagerEventType.java | 3 +-
.../server/resourcemanager/rmapp/RMApp.java | 10 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 11 +-
.../rmapp/RMAppNodeUpdateEvent.java | 9 +-
.../server/resourcemanager/rmnode/RMNode.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 5 +
.../yarn/server/resourcemanager/MockRM.java | 15 +++
.../resourcemanager/TestClientRMService.java | 50 ++++++++
.../TestDecommissioningNodesWatcher.java | 4 +-
.../resourcemanager/TestRMNodeTransitions.java | 13 ++-
.../TestResourceTrackerService.java | 116 ++++++++++++++++++-
.../applicationsmanager/MockAsm.java | 4 +-
.../TestAMRMRPCNodeUpdates.java | 51 ++++++++
.../server/resourcemanager/rmapp/MockRMApp.java | 4 +-
26 files changed, 495 insertions(+), 111 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 885a3b4..3a80641 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -53,7 +53,8 @@ public abstract class NodeReport {
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, String healthReport, long lastHealthReportTime) {
return newInstance(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime, null);
+ capability, numContainers, healthReport, lastHealthReportTime,
+ null, null, null);
}
@Private
@@ -61,7 +62,8 @@ public abstract class NodeReport {
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels) {
+ Set<String> nodeLabels, Integer decommissioningTimeout,
+ NodeUpdateType nodeUpdateType) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
@@ -73,6 +75,8 @@ public abstract class NodeReport {
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
+ nodeReport.setDecommissioningTimeout(decommissioningTimeout);
+ nodeReport.setNodeUpdateType(nodeUpdateType);
return nodeReport;
}
@@ -186,8 +190,8 @@ public abstract class NodeReport {
public abstract void setLastHealthReportTime(long lastHealthReport);
/**
- * Get labels of this node
- * @return labels of this node
+ * Get labels of this node.
+ * @return labels of this node.
*/
@Public
@Stable
@@ -198,8 +202,8 @@ public abstract class NodeReport {
public abstract void setNodeLabels(Set<String> nodeLabels);
/**
- * Get containers aggregated resource utilization in a node
- * @return containers resource utilization
+ * Get containers aggregated resource utilization in a node.
+ * @return containers resource utilization.
*/
@Public
@Stable
@@ -217,8 +221,8 @@ public abstract class NodeReport {
}
/**
- * Get node resource utilization
- * @return node resource utilization
+ * Get node resource utilization.
+ * @return node resource utilization.
*/
@Public
@Stable
@@ -227,4 +231,31 @@ public abstract class NodeReport {
@Private
@Unstable
public abstract void setNodeUtilization(ResourceUtilization nodeUtilization);
+
+ /**
+ * Optional decommissioning timeout in seconds (null indicates absent
+ * timeout).
+ * @return the decommissioning timeout in second.
+ */
+ public Integer getDecommissioningTimeout() {
+ return null;
+ }
+
+ /**
+ * Set the decommissioning timeout in seconds (null indicates absent timeout).
+ * */
+ public void setDecommissioningTimeout(Integer decommissioningTimeout) {}
+
+ /**
+ * Optional node update type (null indicates absent update type).
+ * @return the node update.
+ */
+ public NodeUpdateType getNodeUpdateType() {
+ return NodeUpdateType.NODE_UNUSABLE;
+ }
+
+ /**
+ * Set the node update type (null indicates absent node update type).
+ * */
+ public void setNodeUpdateType(NodeUpdateType nodeUpdateType) {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeUpdateType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeUpdateType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeUpdateType.java
new file mode 100644
index 0000000..9152a6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeUpdateType.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+/**
+ * <p>Taxonomy of the <code>NodeState</code> that a
+ * <code>Node</code> might transition into.</p>
+ * */
+public enum NodeUpdateType {
+ NODE_USABLE,
+ NODE_UNUSABLE,
+ NODE_DECOMMISSIONING
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 7769c48..fdbe2d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -332,6 +332,12 @@ message NodeIdProto {
optional int32 port = 2;
}
+enum NodeUpdateTypeProto {
+ NODE_USABLE = 0;
+ NODE_UNUSABLE = 1;
+ NODE_DECOMMISSIONING = 2;
+}
+
message NodeReportProto {
optional NodeIdProto nodeId = 1;
optional string httpAddress = 2;
@@ -345,6 +351,8 @@ message NodeReportProto {
repeated string node_labels = 10;
optional ResourceUtilizationProto containers_utilization = 11;
optional ResourceUtilizationProto node_utilization = 12;
+ optional uint32 decommissioning_timeout = 13;
+ optional NodeUpdateTypeProto node_update_type = 14;
}
message NodeIdToLabelsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index f4005e9..54537ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -618,7 +618,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
}
public ApplicationReport createFakeAppReport() {
- ApplicationId appId = ApplicationId.newInstance(1000l, 1);
+ ApplicationId appId = ApplicationId.newInstance(1000L, 1);
ApplicationAttemptId attemptId =
ApplicationAttemptId.newInstance(appId, 1);
// create a fake application report
@@ -626,7 +626,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
ApplicationReport.newInstance(appId, attemptId, "fakeUser",
"fakeQueue", "fakeApplicationName", "localhost", 0, null,
YarnApplicationState.FINISHED, "fake an application report", "",
- 1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f,
+ 1000L, 1200L, FinalApplicationStatus.FAILED, null, "", 50f,
"fakeApplicationType", null);
return report;
}
@@ -638,7 +638,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
}
public ApplicationId createFakeAppId() {
- return ApplicationId.newInstance(1000l, 1);
+ return ApplicationId.newInstance(1000L, 1);
}
public ApplicationAttemptId createFakeApplicationAttemptId() {
@@ -657,7 +657,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
NodeId nodeId = NodeId.newInstance("localhost", 0);
NodeReport report =
NodeReport.newInstance(nodeId, NodeState.RUNNING, "localhost",
- "rack1", null, null, 4, null, 1000l, null);
+ "rack1", null, null, 4, null, 1000L);
List<NodeReport> reports = new ArrayList<NodeReport>();
reports.add(report);
return reports;
@@ -680,8 +680,8 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public ApplicationAttemptReport createFakeApplicationAttemptReport() {
return ApplicationAttemptReport.newInstance(
createFakeApplicationAttemptId(), "localhost", 0, "", "", "",
- YarnApplicationAttemptState.RUNNING, createFakeContainerId(), 1000l,
- 1200l);
+ YarnApplicationAttemptState.RUNNING, createFakeContainerId(), 1000L,
+ 1200L);
}
public List<ApplicationAttemptReport>
@@ -694,7 +694,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public ContainerReport createFakeContainerReport() {
return ContainerReport.newInstance(createFakeContainerId(), null,
- NodeId.newInstance("localhost", 0), null, 1000l, 1200l, "", "", 0,
+ NodeId.newInstance("localhost", 0), null, 1000L, 1200L, "", "", 0,
ContainerState.COMPLETE,
"http://" + NodeId.newInstance("localhost", 0).toString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index da1659f..54bd71e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1992,7 +1992,7 @@ public class TestYarnCLI {
NodeReport nodeReport = NodeReport.newInstance(NodeId
.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
"rack1", Records.newRecord(Resource.class), Records
- .newRecord(Resource.class), 0, "", 0, nodeLabels);
+ .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
if (!emptyResourceUtilization) {
ResourceUtilization containersUtilization = ResourceUtilization
.newInstance(1024, 2048, 4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index 0d205e9..ced588d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
@@ -150,8 +151,9 @@ public class NodeReportPBImpl extends NodeReport {
@Override
public void setNodeId(NodeId nodeId) {
maybeInitBuilder();
- if (nodeId == null)
+ if (nodeId == null) {
builder.clearNodeId();
+ }
this.nodeId = nodeId;
}
@@ -177,8 +179,9 @@ public class NodeReportPBImpl extends NodeReport {
@Override
public void setCapability(Resource capability) {
maybeInitBuilder();
- if (capability == null)
+ if (capability == null) {
builder.clearCapability();
+ }
this.capability = capability;
}
@@ -215,8 +218,9 @@ public class NodeReportPBImpl extends NodeReport {
@Override
public void setUsed(Resource used) {
maybeInitBuilder();
- if (used == null)
+ if (used == null) {
builder.clearUsed();
+ }
this.used = used;
}
@@ -234,8 +238,9 @@ public class NodeReportPBImpl extends NodeReport {
@Override
public boolean equals(Object other) {
- if (other == null)
+ if (other == null) {
return false;
+ }
if (other.getClass().isAssignableFrom(this.getClass())) {
return this.getProto().equals(this.getClass().cast(other).getProto());
}
@@ -278,8 +283,9 @@ public class NodeReportPBImpl extends NodeReport {
}
private void mergeLocalToProto() {
- if (viaProto)
+ if (viaProto) {
maybeInitBuilder();
+ }
mergeLocalToBuilder();
proto = builder.build();
viaProto = true;
@@ -387,4 +393,38 @@ public class NodeReportPBImpl extends NodeReport {
}
this.nodeUtilization = nodeResourceUtilization;
}
+
+ @Override
+ public Integer getDecommissioningTimeout() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasDecommissioningTimeout())
+ ? p.getDecommissioningTimeout() : null;
+ }
+
+ @Override
+ public void setDecommissioningTimeout(Integer decommissioningTimeout) {
+ maybeInitBuilder();
+ if (decommissioningTimeout == null || decommissioningTimeout < 0) {
+ builder.clearDecommissioningTimeout();
+ return;
+ }
+ builder.setDecommissioningTimeout(decommissioningTimeout);
+ }
+
+ @Override
+ public NodeUpdateType getNodeUpdateType() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasNodeUpdateType()) ?
+ ProtoUtils.convertFromProtoFormat(p.getNodeUpdateType()) : null;
+ }
+
+ @Override
+ public void setNodeUpdateType(NodeUpdateType nodeUpdateType) {
+ maybeInitBuilder();
+ if (nodeUpdateType == null) {
+ builder.clearNodeUpdateType();
+ return;
+ }
+ builder.setNodeUpdateType(ProtoUtils.convertToProtoFormat(nodeUpdateType));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 528cf8e..f3e665b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
@@ -80,6 +81,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypesProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeUpdateTypeProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.ContainerUpdateTypeProto;
import org.apache.hadoop.yarn.server.api.ContainerType;
@@ -344,6 +346,16 @@ public class ProtoUtils {
}
/*
+ * NodeUpdateType
+ */
+ public static NodeUpdateTypeProto convertToProtoFormat(NodeUpdateType e) {
+ return NodeUpdateTypeProto.valueOf(e.name());
+ }
+ public static NodeUpdateType convertFromProtoFormat(NodeUpdateTypeProto e) {
+ return NodeUpdateType.valueOf(e.name());
+ }
+
+ /*
* ExecutionType
*/
public static ExecutionTypeProto convertToProtoFormat(ExecutionType e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index fec2681..83f912f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -187,23 +188,26 @@ public class BuilderUtils {
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, String healthReport, long lastHealthReportTime) {
return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime, null);
+ capability, numContainers, healthReport, lastHealthReportTime,
+ null, null, null);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels) {
+ Set<String> nodeLabels, Integer decommissioningTimeout,
+ NodeUpdateType nodeUpdateType) {
return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
capability, numContainers, healthReport, lastHealthReportTime,
- nodeLabels, null, null);
+ nodeLabels, null, null, decommissioningTimeout, nodeUpdateType);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, String healthReport, long lastHealthReportTime,
Set<String> nodeLabels, ResourceUtilization containersUtilization,
- ResourceUtilization nodeUtilization) {
+ ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
+ NodeUpdateType nodeUpdateType) {
NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
@@ -217,6 +221,8 @@ public class BuilderUtils {
nodeReport.setNodeLabels(nodeLabels);
nodeReport.setAggregatedContainersUtilization(containersUtilization);
nodeReport.setNodeUtilization(nodeUtilization);
+ nodeReport.setDecommissioningTimeout(decommissioningTimeout);
+ nodeReport.setNodeUpdateType(nodeUpdateType);
return nodeReport;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index e9c6eac..55a3f0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1024,7 +1024,7 @@ public class ClientRMService extends AbstractService implements
return response;
}
- private NodeReport createNodeReports(RMNode rmNode) {
+ private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
Resource used = BuilderUtils.newResource(0, 0);
@@ -1040,7 +1040,8 @@ public class ClientRMService extends AbstractService implements
rmNode.getTotalCapability(), numContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
- rmNode.getNodeUtilization());
+ rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),
+ null);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
index 9631803..ca3eb79 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DecommissioningNodesWatcher.java
@@ -72,11 +72,6 @@ public class DecommissioningNodesWatcher {
private final RMContext rmContext;
- // Default timeout value in mills.
- // Negative value indicates no timeout. 0 means immediate.
- private long defaultTimeoutMs =
- 1000L * YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT;
-
// Once a RMNode is observed in DECOMMISSIONING state,
// All its ContainerStatus update are tracked inside DecomNodeContext.
class DecommissioningNodeContext {
@@ -105,16 +100,15 @@ public class DecommissioningNodesWatcher {
private long lastUpdateTime;
- public DecommissioningNodeContext(NodeId nodeId) {
+ public DecommissioningNodeContext(NodeId nodeId, int timeoutSec) {
this.nodeId = nodeId;
this.appIds = new HashSet<ApplicationId>();
this.decommissioningStartTime = mclock.getTime();
- this.timeoutMs = defaultTimeoutMs;
+ this.timeoutMs = 1000L * timeoutSec;
}
- void updateTimeout(Integer timeoutSec) {
- this.timeoutMs = (timeoutSec == null)?
- defaultTimeoutMs : (1000L * timeoutSec);
+ void updateTimeout(int timeoutSec) {
+ this.timeoutMs = 1000L * timeoutSec;
}
}
@@ -132,7 +126,6 @@ public class DecommissioningNodesWatcher {
}
public void init(Configuration conf) {
- readDecommissioningTimeout(conf);
int v = conf.getInt(
YarnConfiguration.RM_DECOMMISSIONING_NODES_WATCHER_POLL_INTERVAL,
YarnConfiguration
@@ -162,7 +155,8 @@ public class DecommissioningNodesWatcher {
}
} else if (rmNode.getState() == NodeState.DECOMMISSIONING) {
if (context == null) {
- context = new DecommissioningNodeContext(rmNode.getNodeID());
+ context = new DecommissioningNodeContext(rmNode.getNodeID(),
+ rmNode.getDecommissioningTimeout());
decomNodes.put(rmNode.getNodeID(), context);
context.nodeState = rmNode.getState();
context.decommissionedTime = 0;
@@ -416,24 +410,4 @@ public class DecommissioningNodesWatcher {
LOG.debug("Decommissioning node: " + sb.toString());
}
}
-
- // Read possible new DECOMMISSIONING_TIMEOUT_KEY from yarn-site.xml.
- // This enables DecommissioningNodesWatcher to pick up new value
- // without ResourceManager restart.
- private void readDecommissioningTimeout(Configuration conf) {
- try {
- if (conf == null) {
- conf = new YarnConfiguration();
- }
- int v = conf.getInt(
- YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT,
- YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT);
- if (defaultTimeoutMs != 1000L * v) {
- defaultTimeoutMs = 1000L * v;
- LOG.info("Use new decommissioningTimeoutMs: " + defaultTimeoutMs);
- }
- } catch (Exception e) {
- LOG.info("Error readDecommissioningTimeout ", e);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 9774a1a..4c2531e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.PreemptionContainer;
import org.apache.hadoop.yarn.api.records.PreemptionContract;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
@@ -83,6 +84,8 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Set;
/**
@@ -326,10 +329,12 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
}
private void handleNodeUpdates(RMApp app, AllocateResponse allocateResponse) {
- List<RMNode> updatedNodes = new ArrayList<>();
+ Map<RMNode, NodeUpdateType> updatedNodes = new HashMap<>();
if(app.pullRMNodeUpdates(updatedNodes) > 0) {
List<NodeReport> updatedNodeReports = new ArrayList<>();
- for(RMNode rmNode: updatedNodes) {
+ for(Map.Entry<RMNode, NodeUpdateType> rmNodeEntry :
+ updatedNodes.entrySet()) {
+ RMNode rmNode = rmNodeEntry.getKey();
SchedulerNodeReport schedulerNodeReport =
getScheduler().getNodeReport(rmNode.getNodeID());
Resource used = BuilderUtils.newResource(0, 0);
@@ -344,7 +349,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
rmNode.getHttpAddress(), rmNode.getRackName(), used,
rmNode.getTotalCapability(), numContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
- rmNode.getNodeLabels());
+ rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
+ rmNodeEntry.getValue());
updatedNodeReports.add(report);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
index edd173b..647dfa3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManager.java
@@ -72,6 +72,11 @@ public class NodesListManager extends CompositeService implements
private Configuration conf;
private final RMContext rmContext;
+ // Default decommissioning timeout value in seconds.
+ // Negative value indicates no timeout. 0 means immediate.
+ private int defaultDecTimeoutSecs =
+ YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT;
+
private String includesFile;
private String excludesFile;
@@ -214,6 +219,11 @@ public class NodesListManager extends CompositeService implements
private void refreshHostsReader(
Configuration yarnConf, boolean graceful, Integer timeout)
throws IOException, YarnException {
+ // resolve the default timeout to the decommission timeout that is
+ // configured at this moment
+ if (null == timeout) {
+ timeout = readDecommissioningTimeout(yarnConf);
+ }
if (null == yarnConf) {
yarnConf = new YarnConfiguration();
}
@@ -252,7 +262,7 @@ public class NodesListManager extends CompositeService implements
// Gracefully decommission excluded nodes that are not already
// DECOMMISSIONED nor DECOMMISSIONING; Take no action for excluded nodes
// that are already DECOMMISSIONED or DECOMMISSIONING.
- private void handleExcludeNodeList(boolean graceful, Integer timeout) {
+ private void handleExcludeNodeList(boolean graceful, int timeout) {
// DECOMMISSIONED/DECOMMISSIONING nodes need to be re-commissioned.
List<RMNode> nodesToRecom = new ArrayList<RMNode>();
@@ -463,36 +473,40 @@ public class NodesListManager extends CompositeService implements
&& !(excludeList.contains(hostName) || excludeList.contains(ip));
}
+ private void sendRMAppNodeUpdateEventToNonFinalizedApps(
+ RMNode eventNode, RMAppNodeUpdateType appNodeUpdateType) {
+ for(RMApp app : rmContext.getRMApps().values()) {
+ if (!app.isAppFinalStateStored()) {
+ this.rmContext
+ .getDispatcher()
+ .getEventHandler()
+ .handle(
+ new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
+ appNodeUpdateType));
+ }
+ }
+ }
+
@Override
public void handle(NodesListManagerEvent event) {
RMNode eventNode = event.getNode();
switch (event.getType()) {
case NODE_UNUSABLE:
LOG.debug(eventNode + " reported unusable");
- for(RMApp app: rmContext.getRMApps().values()) {
- if (!app.isAppFinalStateStored()) {
- this.rmContext
- .getDispatcher()
- .getEventHandler()
- .handle(
- new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
- RMAppNodeUpdateType.NODE_UNUSABLE));
- }
- }
+ sendRMAppNodeUpdateEventToNonFinalizedApps(eventNode,
+ RMAppNodeUpdateType.NODE_UNUSABLE);
break;
case NODE_USABLE:
LOG.debug(eventNode + " reported usable");
- for (RMApp app : rmContext.getRMApps().values()) {
- if (!app.isAppFinalStateStored()) {
- this.rmContext
- .getDispatcher()
- .getEventHandler()
- .handle(
- new RMAppNodeUpdateEvent(app.getApplicationId(), eventNode,
- RMAppNodeUpdateType.NODE_USABLE));
- }
- }
+ sendRMAppNodeUpdateEventToNonFinalizedApps(eventNode,
+ RMAppNodeUpdateType.NODE_USABLE);
+ break;
+ case NODE_DECOMMISSIONING:
+ LOG.debug(eventNode + " reported decommissioning");
+ sendRMAppNodeUpdateEventToNonFinalizedApps(
+ eventNode, RMAppNodeUpdateType.NODE_DECOMMISSIONING);
break;
+
default:
LOG.error("Ignoring invalid eventtype " + event.getType());
}
@@ -611,6 +625,28 @@ public class NodesListManager extends CompositeService implements
}
}
+ // Read possible new DECOMMISSIONING_TIMEOUT_KEY from yarn-site.xml.
+ // This enables NodesListManager to pick up new value without
+ // ResourceManager restart.
+ private int readDecommissioningTimeout(Configuration pConf) {
+ try {
+ if (pConf == null) {
+ pConf = new YarnConfiguration();
+ }
+ int configuredDefaultDecTimeoutSecs =
+ pConf.getInt(YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT,
+ YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT);
+ if (defaultDecTimeoutSecs != configuredDefaultDecTimeoutSecs) {
+ defaultDecTimeoutSecs = configuredDefaultDecTimeoutSecs;
+ LOG.info("Use new decommissioningTimeoutSecs: "
+ + defaultDecTimeoutSecs);
+ }
+ } catch (Exception e) {
+ LOG.warn("Error readDecommissioningTimeout " + e.getMessage());
+ }
+ return defaultDecTimeoutSecs;
+ }
+
/**
* A NodeId instance needed upon startup for populating inactive nodes Map.
* It only knows the hostname/ip and marks the port to -1 or invalid.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManagerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManagerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManagerEventType.java
index 2afc8e6..db16bc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManagerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/NodesListManagerEventType.java
@@ -20,5 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
public enum NodesListManagerEventType {
NODE_USABLE,
- NODE_UNUSABLE
+ NODE_UNUSABLE,
+ NODE_DECOMMISSIONING
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 93c41b6..8583789 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -36,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.CollectorInfo;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -154,10 +154,12 @@ public interface RMApp extends EventHandler<RMAppEvent> {
* received by the RMApp. Updates can be node becoming lost or becoming
* healthy etc. The method clears the information from the {@link RMApp}. So
* each call to this method gives the delta from the previous call.
- * @param updatedNodes Collection into which the updates are transferred
- * @return the number of nodes added to the {@link Collection}
+ * @param updatedNodes Map into which the updates are transferred, with each
+ * node updates as the key, and the {@link NodeUpdateType} for that update
+ * as the corresponding value.
+ * @return the number of nodes added to the {@link Map}
*/
- int pullRMNodeUpdates(Collection<RMNode> updatedNodes);
+ int pullRMNodeUpdates(Map<RMNode, NodeUpdateType> updatedNodes);
/**
* The finish time of the {@link RMApp}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 6896254..0266b83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -20,11 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.net.InetAddress;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -149,7 +148,7 @@ public class RMAppImpl implements RMApp, Recoverable {
private final Map<ApplicationAttemptId, RMAppAttempt> attempts
= new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>();
private final long submitTime;
- private final Set<RMNode> updatedNodes = new HashSet<RMNode>();
+ private final Map<RMNode, NodeUpdateType> updatedNodes = new HashMap<>();
private final String applicationType;
private final Set<String> applicationTags;
@@ -677,11 +676,11 @@ public class RMAppImpl implements RMApp, Recoverable {
}
@Override
- public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
+ public int pullRMNodeUpdates(Map<RMNode, NodeUpdateType> upNodes) {
this.writeLock.lock();
try {
int updatedNodeCount = this.updatedNodes.size();
- updatedNodes.addAll(this.updatedNodes);
+ upNodes.putAll(this.updatedNodes);
this.updatedNodes.clear();
return updatedNodeCount;
} finally {
@@ -987,7 +986,7 @@ public class RMAppImpl implements RMApp, Recoverable {
private void processNodeUpdate(RMAppNodeUpdateType type, RMNode node) {
NodeState nodeState = node.getState();
- updatedNodes.add(node);
+ updatedNodes.put(node, RMAppNodeUpdateType.convertToNodeUpdateType(type));
LOG.debug("Received node update event:" + type + " for node:" + node
+ " with state:" + nodeState);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNodeUpdateEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNodeUpdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNodeUpdateEvent.java
index ba8af98..245d9a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNodeUpdateEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppNodeUpdateEvent.java
@@ -19,13 +19,20 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
public class RMAppNodeUpdateEvent extends RMAppEvent {
public enum RMAppNodeUpdateType {
NODE_USABLE,
- NODE_UNUSABLE
+ NODE_UNUSABLE,
+ NODE_DECOMMISSIONING;
+
+ public static NodeUpdateType convertToNodeUpdateType(
+ RMAppNodeUpdateType rmAppNodeUpdateType) {
+ return NodeUpdateType.valueOf(rmAppNodeUpdateType.name());
+ }
}
private final RMNode node;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index ab15c95..328c040 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -183,7 +183,7 @@ public interface RMNode {
void setUntrackedTimeStamp(long timeStamp);
/*
* Optional decommissioning timeout in second
- * (null indicates default timeout).
+ * (null indicates absent timeout).
* @return the decommissioning timeout in second.
*/
Integer getDecommissioningTimeout();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index d0bfecf..2b013a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -1160,6 +1160,11 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
// Update NM metrics during graceful decommissioning.
rmNode.updateMetricsForGracefulDecommission(initState, finalState);
rmNode.decommissioningTimeout = timeout;
+ // Notify NodesListManager to notify all RMApp so that each
+ // Application Master could take any required actions.
+ rmNode.context.getDispatcher().getEventHandler().handle(
+ new NodesListManagerEvent(
+ NodesListManagerEventType.NODE_DECOMMISSIONING, rmNode));
if (rmNode.originalTotalCapability == null){
rmNode.originalTotalCapability =
Resources.clone(rmNode.totalCapability);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index d128b02..711f008 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeDecommissioningEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
@@ -917,12 +918,26 @@ public class MockRM extends ResourceManager {
node.getState());
}
+ public void sendNodeGracefulDecommission(
+ MockNM nm, int timeout) throws Exception {
+ RMNodeImpl node = (RMNodeImpl)
+ getRMContext().getRMNodes().get(nm.getNodeId());
+ Assert.assertNotNull("node shouldn't be null", node);
+ node.handle(new RMNodeDecommissioningEvent(nm.getNodeId(), timeout));
+ }
+
public void sendNodeEvent(MockNM nm, RMNodeEventType event) throws Exception {
RMNodeImpl node = (RMNodeImpl)
getRMContext().getRMNodes().get(nm.getNodeId());
+ Assert.assertNotNull("node shouldn't be null", node);
node.handle(new RMNodeEvent(nm.getNodeId(), event));
}
+ public Integer getDecommissioningTimeout(NodeId nodeid) {
+ return this.getRMContext().getRMNodes()
+ .get(nodeid).getDecommissioningTimeout();
+ }
+
public KillApplicationResponse killApp(ApplicationId appId) throws Exception {
ApplicationClientProtocol client = getClientRMService();
KillApplicationRequest req = KillApplicationRequest.newInstance(appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index ae57dfb..1c50dd3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -181,6 +181,50 @@ public class TestClientRMService {
private final static String QUEUE_2 = "Q-2";
@Test
+ public void testGetDecommissioningClusterNodes() throws Exception {
+ MockRM rm = new MockRM() {
+ protected ClientRMService createClientRMService() {
+ return new ClientRMService(this.rmContext, scheduler,
+ this.rmAppManager, this.applicationACLsManager,
+ this.queueACLsManager,
+ this.getRMContext().getRMDelegationTokenSecretManager());
+ };
+ };
+ rm.start();
+
+ int nodeMemory = 1024;
+ MockNM nm1 = rm.registerNode("host1:1234", nodeMemory);
+ rm.sendNodeStarted(nm1);
+ nm1.nodeHeartbeat(true);
+ rm.waitForState(nm1.getNodeId(), NodeState.RUNNING);
+ Integer decommissioningTimeout = 600;
+ rm.sendNodeGracefulDecommission(nm1, decommissioningTimeout);
+ rm.waitForState(nm1.getNodeId(), NodeState.DECOMMISSIONING);
+
+ // Create a client.
+ Configuration conf = new Configuration();
+ YarnRPC rpc = YarnRPC.create(conf);
+ InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
+ LOG.info("Connecting to ResourceManager at " + rmAddress);
+ ApplicationClientProtocol client =
+ (ApplicationClientProtocol) rpc
+ .getProxy(ApplicationClientProtocol.class, rmAddress, conf);
+
+ // Make call
+ List<NodeReport> nodeReports = client.getClusterNodes(
+ GetClusterNodesRequest.newInstance(
+ EnumSet.of(NodeState.DECOMMISSIONING)))
+ .getNodeReports();
+ Assert.assertEquals(1, nodeReports.size());
+ NodeReport nr = nodeReports.iterator().next();
+ Assert.assertEquals(decommissioningTimeout, nr.getDecommissioningTimeout());
+ Assert.assertNull(nr.getNodeUpdateType());
+
+ rpc.stopProxy(client, conf);
+ rm.close();
+ }
+
+ @Test
public void testGetClusterNodes() throws Exception {
MockRM rm = new MockRM() {
protected ClientRMService createClientRMService() {
@@ -228,6 +272,8 @@ public class TestClientRMService {
// Check node's label = x
Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("x"));
+ Assert.assertNull(nodeReports.get(0).getDecommissioningTimeout());
+ Assert.assertNull(nodeReports.get(0).getNodeUpdateType());
// Now make the node unhealthy.
node.nodeHeartbeat(false);
@@ -251,6 +297,8 @@ public class TestClientRMService {
nodeReports.get(0).getNodeState());
Assert.assertTrue(nodeReports.get(0).getNodeLabels().contains("y"));
+ Assert.assertNull(nodeReports.get(0).getDecommissioningTimeout());
+ Assert.assertNull(nodeReports.get(0).getNodeUpdateType());
// Remove labels of host1
map = new HashMap<NodeId, Set<String>>();
@@ -267,6 +315,8 @@ public class TestClientRMService {
for (NodeReport report : nodeReports) {
Assert.assertTrue(report.getNodeLabels() != null
&& report.getNodeLabels().isEmpty());
+ Assert.assertNull(report.getDecommissioningTimeout());
+ Assert.assertNull(report.getNodeUpdateType());
}
rpc.stopProxy(client, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
index 690de30..4371156 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestDecommissioningNodesWatcher.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.DecommissioningNodesWatcher
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
@@ -69,7 +68,8 @@ public class TestDecommissioningNodesWatcher {
MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
// Setup nm1 as DECOMMISSIONING for DecommissioningNodesWatcher.
- rm.sendNodeEvent(nm1, RMNodeEventType.GRACEFUL_DECOMMISSION);
+ rm.sendNodeGracefulDecommission(nm1,
+ YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT);
rm.waitForState(id1, NodeState.DECOMMISSIONING);
// Update status with decreasing number of running containers until 0.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index ba806ab..3657123 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -27,7 +27,9 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
import java.util.Random;
@@ -98,13 +100,16 @@ public class TestRMNodeTransitions {
}
private NodesListManagerEvent nodesListManagerEvent = null;
-
+ private List<NodeState> nodesListManagerEventsNodeStateSequence =
+ new LinkedList<>();
+
private class TestNodeListManagerEventDispatcher implements
EventHandler<NodesListManagerEvent> {
@Override
public void handle(NodesListManagerEvent event) {
nodesListManagerEvent = event;
+ nodesListManagerEventsNodeStateSequence.add(event.getNode().getState());
}
}
@@ -150,7 +155,7 @@ public class TestRMNodeTransitions {
NodeId nodeId = BuilderUtils.newNodeId("localhost", 0);
node = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
nodesListManagerEvent = null;
-
+ nodesListManagerEventsNodeStateSequence.clear();
}
@After
@@ -721,6 +726,8 @@ public class TestRMNodeTransitions {
node.handle(new RMNodeEvent(node.getNodeID(),
RMNodeEventType.GRACEFUL_DECOMMISSION));
Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
+ Assert.assertEquals(Arrays.asList(NodeState.NEW, NodeState.RUNNING),
+ nodesListManagerEventsNodeStateSequence);
Assert
.assertEquals("Active Nodes", initialActive - 1, cm.getNumActiveNMs());
Assert.assertEquals("Decommissioning Nodes", initialDecommissioning + 1,
@@ -1008,7 +1015,7 @@ public class TestRMNodeTransitions {
Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
Assert.assertNotNull(nodesListManagerEvent);
- Assert.assertEquals(NodesListManagerEventType.NODE_USABLE,
+ Assert.assertEquals(NodesListManagerEventType.NODE_DECOMMISSIONING,
nodesListManagerEvent.getType());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 1cb2d0d..fc6326e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -39,7 +39,14 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
-
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.dom.DOMSource;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.OutputKeys;
+
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.MetricsSystem;
@@ -99,14 +106,19 @@ import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.junit.After;
import org.junit.Assert;
import org.junit.Test;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
public class TestResourceTrackerService extends NodeLabelTestBase {
private final static File TEMP_DIR = new File(System.getProperty(
"test.build.data", "/tmp"), "decommision");
- private final File hostFile = new File(TEMP_DIR + File.separator + "hostFile.txt");
+ private final File hostFile =
+ new File(TEMP_DIR + File.separator + "hostFile.txt");
private final File excludeHostFile = new File(TEMP_DIR + File.separator +
"excludeHostFile.txt");
+ private final File excludeHostXmlFile =
+ new File(TEMP_DIR + File.separator + "excludeHostFile.xml");
private MockRM rm;
@@ -291,6 +303,67 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
Assert.assertEquals(NodeAction.SHUTDOWN, nodeHeartbeat3.getNodeAction());
}
+ @Test
+ public void testGracefulDecommissionDefaultTimeoutResolution()
+ throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, excludeHostXmlFile
+ .getAbsolutePath());
+
+ writeToHostsXmlFile(excludeHostXmlFile, Pair.of("", null));
+ rm = new MockRM(conf);
+ rm.start();
+
+ int nodeMemory = 1024;
+ MockNM nm1 = rm.registerNode("host1:1234", nodeMemory);
+ MockNM nm2 = rm.registerNode("host2:5678", nodeMemory);
+ MockNM nm3 = rm.registerNode("host3:9101", nodeMemory);
+
+ NodeHeartbeatResponse nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat3 = nm3.nodeHeartbeat(true);
+
+ Assert.assertTrue(
+ NodeAction.NORMAL.equals(nodeHeartbeat1.getNodeAction()));
+ Assert.assertTrue(
+ NodeAction.NORMAL.equals(nodeHeartbeat2.getNodeAction()));
+ Assert.assertTrue(
+ NodeAction.NORMAL.equals(nodeHeartbeat3.getNodeAction()));
+
+ rm.waitForState(nm1.getNodeId(), NodeState.RUNNING);
+ rm.waitForState(nm2.getNodeId(), NodeState.RUNNING);
+ rm.waitForState(nm3.getNodeId(), NodeState.RUNNING);
+
+ // Graceful decommission both host1 and host2, with
+ // non default timeout for host1
+ final Integer nm1DecommissionTimeout = 20;
+ writeToHostsXmlFile(
+ excludeHostXmlFile,
+ Pair.of(nm1.getNodeId().getHost(), nm1DecommissionTimeout),
+ Pair.of(nm2.getNodeId().getHost(), null));
+ rm.getNodesListManager().refreshNodes(conf, true);
+ rm.waitForState(nm1.getNodeId(), NodeState.DECOMMISSIONING);
+ rm.waitForState(nm2.getNodeId(), NodeState.DECOMMISSIONING);
+ Assert.assertEquals(
+ nm1DecommissionTimeout, rm.getDecommissioningTimeout(nm1.getNodeId()));
+ Integer defaultDecTimeout =
+ conf.getInt(YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT,
+ YarnConfiguration.DEFAULT_RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT);
+ Assert.assertEquals(
+ defaultDecTimeout, rm.getDecommissioningTimeout(nm2.getNodeId()));
+
+ // Graceful decommission host3 with a new default timeout
+ final Integer newDefaultDecTimeout = defaultDecTimeout + 10;
+ writeToHostsXmlFile(
+ excludeHostXmlFile, Pair.of(nm3.getNodeId().getHost(), null));
+ conf.setInt(YarnConfiguration.RM_NODE_GRACEFUL_DECOMMISSION_TIMEOUT,
+ newDefaultDecTimeout);
+ rm.getNodesListManager().refreshNodes(conf, true);
+ rm.waitForState(nm3.getNodeId(), NodeState.DECOMMISSIONING);
+ Assert.assertEquals(
+ newDefaultDecTimeout, rm.getDecommissioningTimeout(nm3.getNodeId()));
+ }
+
/**
* Graceful decommission node with running application.
*/
@@ -1967,16 +2040,20 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
rm.stop();
}
+ private void ensureFileExists(File file) throws IOException {
+ if (!file.exists()) {
+ TEMP_DIR.mkdirs();
+ file.createNewFile();
+ }
+ }
+
private void writeToHostsFile(String... hosts) throws IOException {
writeToHostsFile(hostFile, hosts);
}
private void writeToHostsFile(File file, String... hosts)
throws IOException {
- if (!file.exists()) {
- TEMP_DIR.mkdirs();
- file.createNewFile();
- }
+ ensureFileExists(file);
FileOutputStream fStream = null;
try {
fStream = new FileOutputStream(file);
@@ -1992,6 +2069,33 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
}
+ private void writeToHostsXmlFile(
+ File file, Pair<String, Integer>... hostsAndTimeouts) throws Exception {
+ ensureFileExists(file);
+ DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
+ Document doc = dbFactory.newDocumentBuilder().newDocument();
+ Element hosts = doc.createElement("hosts");
+ doc.appendChild(hosts);
+ for (Pair<String, Integer> hostsAndTimeout : hostsAndTimeouts) {
+ Element host = doc.createElement("host");
+ hosts.appendChild(host);
+ Element name = doc.createElement("name");
+ host.appendChild(name);
+ name.appendChild(doc.createTextNode(hostsAndTimeout.getLeft()));
+ if (hostsAndTimeout.getRight() != null) {
+ Element timeout = doc.createElement("timeout");
+ host.appendChild(timeout);
+ timeout.appendChild(
+ doc.createTextNode(hostsAndTimeout.getRight().toString())
+ );
+ }
+ }
+ TransformerFactory transformerFactory = TransformerFactory.newInstance();
+ Transformer transformer = transformerFactory.newTransformer();
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");
+ transformer.transform(new DOMSource(doc), new StreamResult(file));
+ }
+
private void checkDecommissionedNMCount(MockRM rm, int count)
throws InterruptedException {
int waitCount = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 399df02..9ef48db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager;
-import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -38,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -154,7 +154,7 @@ public abstract class MockAsm extends MockApps {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
+ public int pullRMNodeUpdates(Map<RMNode, NodeUpdateType> updatedNodes) {
throw new UnsupportedOperationException("Not supported yet.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
index f9f0b74..dd9bcd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
@@ -82,6 +83,13 @@ public class TestAMRMRPCNodeUpdates {
rm.drainEvents();
}
+ private void syncNodeGracefulDecommission(
+ MockNM nm, int timeout) throws Exception {
+ rm.sendNodeGracefulDecommission(nm, timeout);
+ rm.waitForState(nm.getNodeId(), NodeState.DECOMMISSIONING);
+ rm.drainEvents();
+ }
+
private AllocateResponse allocate(final ApplicationAttemptId attemptId,
final AllocateRequest req) throws Exception {
UserGroupInformation ugi =
@@ -99,6 +107,39 @@ public class TestAMRMRPCNodeUpdates {
}
@Test
+ public void testAMRMDecommissioningNodes() throws Exception {
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10000);
+ MockNM nm2 = rm.registerNode("127.0.0.2:1234", 10000);
+ rm.drainEvents();
+
+ RMApp app1 = rm.submitApp(2000);
+
+ // Trigger the scheduling so the AM gets 'launched' on nm1
+ nm1.nodeHeartbeat(true);
+
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
+
+ // register AM returns no unusable node
+ am1.registerAppAttempt();
+
+ Integer decommissioningTimeout = 600;
+ syncNodeGracefulDecommission(nm2, decommissioningTimeout);
+
+ AllocateRequest allocateRequest1 =
+ AllocateRequest.newInstance(0, 0F, null, null, null);
+ AllocateResponse response1 =
+ allocate(attempt1.getAppAttemptId(), allocateRequest1);
+ List<NodeReport> updatedNodes = response1.getUpdatedNodes();
+ Assert.assertEquals(1, updatedNodes.size());
+ NodeReport nr = updatedNodes.iterator().next();
+ Assert.assertEquals(
+ decommissioningTimeout, nr.getDecommissioningTimeout());
+ Assert.assertEquals(
+ NodeUpdateType.NODE_DECOMMISSIONING, nr.getNodeUpdateType());
+ }
+
+ @Test
public void testAMRMUnusableNodes() throws Exception {
MockNM nm1 = rm.registerNode("127.0.0.1:1234", 10000);
@@ -138,6 +179,8 @@ public class TestAMRMRPCNodeUpdates {
NodeReport nr = updatedNodes.iterator().next();
Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
+ Assert.assertNull(nr.getDecommissioningTimeout());
+ Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType());
// resending the allocate request returns the same result
response1 = allocate(attempt1.getAppAttemptId(), allocateRequest1);
@@ -146,6 +189,8 @@ public class TestAMRMRPCNodeUpdates {
nr = updatedNodes.iterator().next();
Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
Assert.assertEquals(NodeState.UNHEALTHY, nr.getNodeState());
+ Assert.assertNull(nr.getDecommissioningTimeout());
+ Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType());
syncNodeLost(nm3);
@@ -159,6 +204,8 @@ public class TestAMRMRPCNodeUpdates {
nr = updatedNodes.iterator().next();
Assert.assertEquals(nm3.getNodeId(), nr.getNodeId());
Assert.assertEquals(NodeState.LOST, nr.getNodeState());
+ Assert.assertNull(nr.getDecommissioningTimeout());
+ Assert.assertEquals(NodeUpdateType.NODE_UNUSABLE, nr.getNodeUpdateType());
// registering another AM gives it the complete failed list
RMApp app2 = rm.submitApp(2000);
@@ -190,6 +237,8 @@ public class TestAMRMRPCNodeUpdates {
nr = updatedNodes.iterator().next();
Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
+ Assert.assertNull(nr.getDecommissioningTimeout());
+ Assert.assertEquals(NodeUpdateType.NODE_USABLE, nr.getNodeUpdateType());
allocateRequest2 =
AllocateRequest.newInstance(response2.getResponseId(), 0F, null, null,
@@ -200,6 +249,8 @@ public class TestAMRMRPCNodeUpdates {
nr = updatedNodes.iterator().next();
Assert.assertEquals(nm4.getNodeId(), nr.getNodeId());
Assert.assertEquals(NodeState.RUNNING, nr.getNodeState());
+ Assert.assertNull(nr.getDecommissioningTimeout());
+ Assert.assertEquals(NodeUpdateType.NODE_USABLE, nr.getNodeUpdateType());
// subsequent allocate calls should return no updated nodes
allocateRequest2 =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ca7e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 39a7f99..6c64a67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
@@ -36,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.CollectorInfo;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.LogAggregationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeUpdateType;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -235,7 +235,7 @@ public class MockRMApp implements RMApp {
}
@Override
- public int pullRMNodeUpdates(Collection<RMNode> updatedNodes) {
+ public int pullRMNodeUpdates(Map<RMNode, NodeUpdateType> updatedNodes) {
throw new UnsupportedOperationException("Not supported yet.");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/49] hadoop git commit: YARN-7531. ResourceRequest.equal does not
check ExecutionTypeRequest.enforceExecutionType().
Posted by su...@apache.org.
YARN-7531. ResourceRequest.equal does not check ExecutionTypeRequest.enforceExecutionType().
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/67bbbe1c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/67bbbe1c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/67bbbe1c
Branch: refs/heads/YARN-5881
Commit: 67bbbe1c0c05fa01b08a8dabe93c146935420450
Parents: 0ed44f2
Author: Haibo Chen <ha...@apache.org>
Authored: Fri Nov 17 14:30:43 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Tue Nov 21 09:09:16 2017 -0800
----------------------------------------------------------------------
.../yarn/api/records/ResourceRequest.java | 3 +-
.../hadoop/yarn/api/TestResourceRequest.java | 47 ++++++++++++++++++++
2 files changed, 48 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67bbbe1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index beb3380..e46647a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -630,8 +630,7 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
if (other.getExecutionTypeRequest() != null) {
return false;
}
- } else if (!execTypeRequest.getExecutionType()
- .equals(other.getExecutionTypeRequest().getExecutionType())) {
+ } else if (!execTypeRequest.equals(other.getExecutionTypeRequest())) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/67bbbe1c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java
new file mode 100644
index 0000000..aef838c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestResourceRequest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * The class to test {@link ResourceRequest}.
+ */
+public class TestResourceRequest {
+
+ @Test
+ public void testEqualsOnExecutionTypeRequest() {
+ ResourceRequest resourceRequestA =
+ ResourceRequest.newInstance(Priority.newInstance(0), "localhost",
+ Resource.newInstance(1024, 1), 1, false, "",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+
+ ResourceRequest resourceRequestB =
+ ResourceRequest.newInstance(Priority.newInstance(0), "localhost",
+ Resource.newInstance(1024, 1), 1, false, "",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, false));
+
+ Assert.assertFalse(resourceRequestA.equals(resourceRequestB));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/49] hadoop git commit: HADOOP-15024 Support user agent
configuration and include that & Hadoop version information to oss server.
Contributed by Sammi Chen.
Posted by su...@apache.org.
HADOOP-15024 Support user agent configuration and include that & Hadoop version information to oss server.
Contributed by Sammi Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c326fc89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c326fc89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c326fc89
Branch: refs/heads/YARN-5881
Commit: c326fc89b06a8fe0978306378ba217748c7f2054
Parents: 9fb4eff
Author: Steve Loughran <st...@apache.org>
Authored: Mon Nov 20 18:56:42 2017 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Mon Nov 20 18:56:42 2017 +0000
----------------------------------------------------------------------
.../apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java | 4 ++++
.../main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java | 7 +++++++
.../src/site/markdown/tools/hadoop-aliyun/index.md | 2 +-
3 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c326fc89/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index 2e8edc7..a7f13c0 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.util.VersionInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,6 +102,9 @@ public class AliyunOSSFileSystemStore {
ESTABLISH_TIMEOUT_DEFAULT));
clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY,
SOCKET_TIMEOUT_DEFAULT));
+ clientConf.setUserAgent(
+ conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/"
+ + VersionInfo.getVersion());
String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c326fc89/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 04a2ccd..baa171f 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.fs.aliyun.oss;
+import com.aliyun.oss.common.utils.VersionInfoUtils;
+
/**
* ALL configuration constants for OSS filesystem.
*/
@@ -26,6 +28,11 @@ public final class Constants {
private Constants() {
}
+ // User agent
+ public static final String USER_AGENT_PREFIX = "fs.oss.user.agent.prefix";
+ public static final String USER_AGENT_PREFIX_DEFAULT =
+ VersionInfoUtils.getDefaultUserAgent();
+
// Class of credential provider
public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY =
"fs.oss.credentials.provider";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c326fc89/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
index 2913279..9f24ce6 100644
--- a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -274,7 +274,7 @@ XInclude inclusion. Here is an example of `contract-test-options.xml`:
<property>
<name>fs.oss.impl</name>
- <value>org.apache.hadoop.fs.aliyun.AliyunOSSFileSystem</value>
+ <value>org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem</value>
</property>
<property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/49] hadoop git commit: YARN-6471. Support to add min/max resource
configuration for a queue. (Sunil G via wangda)
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
index 0544387..4985a1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.util.resource.Resources;
import java.util.ArrayList;
import java.util.Collections;
@@ -121,6 +123,15 @@ public class PriorityUtilizationQueueOrderingPolicy implements QueueOrderingPoli
// For queue with same used ratio / priority, queue with higher configured
// capacity goes first
if (0 == rc) {
+ Resource minEffRes1 = q1.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ Resource minEffRes2 = q2.getQueueResourceQuotas()
+ .getConfiguredMinResource(p);
+ if (!minEffRes1.equals(Resources.none())
+ && !minEffRes2.equals(Resources.none())) {
+ return minEffRes2.compareTo(minEffRes1);
+ }
+
float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p);
float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p);
return Float.compare(abs2, abs1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
index 22705cc..86b2fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
@@ -62,6 +62,8 @@ public class CapacitySchedulerQueueInfo {
protected long pendingContainers;
protected QueueCapacitiesInfo capacities;
protected ResourcesInfo resources;
+ protected ResourceInfo minEffectiveCapacity;
+ protected ResourceInfo maxEffectiveCapacity;
CapacitySchedulerQueueInfo() {
};
@@ -105,6 +107,11 @@ public class CapacitySchedulerQueueInfo {
ResourceUsage queueResourceUsage = q.getQueueResourceUsage();
populateQueueResourceUsage(queueResourceUsage);
+
+ minEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMinResource());
+ maxEffectiveCapacity = new ResourceInfo(
+ q.getQueueResourceQuotas().getEffectiveMaxResource());
}
protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) {
@@ -200,4 +207,12 @@ public class CapacitySchedulerQueueInfo {
public ResourcesInfo getResources() {
return resources;
}
+
+ public ResourceInfo getMinEffectiveCapacity(){
+ return minEffectiveCapacity;
+ }
+
+ public ResourceInfo getMaxEffectiveCapacity(){
+ return maxEffectiveCapacity;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
index b5c20f4..909d023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
+import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -142,6 +143,13 @@ public class MockNM {
return this.registeringCollectors;
}
+ public void unRegisterNode() throws Exception {
+ UnRegisterNodeManagerRequest request = Records
+ .newRecord(UnRegisterNodeManagerRequest.class);
+ request.setNodeId(nodeId);
+ resourceTracker.unRegisterNodeManager(request);
+ }
+
public RegisterNodeManagerResponse registerNode() throws Exception {
return registerNode(null, null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 711f008..19ca6d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -823,6 +823,12 @@ public class MockRM extends ResourceManager {
return rmApp;
}
+ public MockNM unRegisterNode(MockNM nm) throws Exception {
+ nm.unRegisterNode();
+ drainEventsImplicitly();
+ return nm;
+ }
+
public MockNM registerNode(String nodeIdStr, int memory) throws Exception {
MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService());
nm.registerNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 0bc5cb5..1dda8a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
@@ -644,9 +645,11 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
QueueCapacities qc = new QueueCapacities(0 == myLevel);
ResourceUsage ru = new ResourceUsage();
+ QueueResourceQuotas qr = new QueueResourceQuotas();
when(queue.getQueueCapacities()).thenReturn(qc);
when(queue.getQueueResourceUsage()).thenReturn(ru);
+ when(queue.getQueueResourceQuotas()).thenReturn(qr);
LOG.debug("Setup queue, name=" + queue.getQueueName() + " path="
+ queue.getQueuePath());
@@ -679,7 +682,17 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
qc.setAbsoluteMaximumCapacity(partitionName, absMax);
qc.setAbsoluteUsedCapacity(partitionName, absUsed);
qc.setUsedCapacity(partitionName, used);
+ qr.setEffectiveMaxResource(parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(parseResourceFromString(values[0].trim()));
+ qr.setEffectiveMaxResource(partitionName,
+ parseResourceFromString(values[1].trim()));
+ qr.setEffectiveMinResource(partitionName,
+ parseResourceFromString(values[0].trim()));
when(queue.getUsedCapacity()).thenReturn(used);
+ when(queue.getEffectiveCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[0].trim()));
+ when(queue.getEffectiveMaxCapacity(partitionName))
+ .thenReturn(parseResourceFromString(values[1].trim()));
ru.setPending(partitionName, pending);
// Setup reserved resource if it contained by input config
Resource reserved = Resources.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 694be09..dd950f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -48,7 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preempti
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy;
import org.apache.hadoop.yarn.util.Clock;
@@ -437,8 +437,8 @@ public class TestProportionalCapacityPreemptionPolicy {
policy.editSchedule();
// queueF(appD) wants resources, Verify that resources come from queueE(appC)
// because it's a sibling and queueB(appA) because queueA is over capacity.
- verify(mDisp, times(28)).handle(argThat(new IsPreemptionRequestFor(appA)));
- verify(mDisp, times(22)).handle(argThat(new IsPreemptionRequestFor(appC)));
+ verify(mDisp, times(27)).handle(argThat(new IsPreemptionRequestFor(appA)));
+ verify(mDisp, times(23)).handle(argThat(new IsPreemptionRequestFor(appC)));
// Need to call setup() again to reset mDisp
setup();
@@ -1202,6 +1202,17 @@ public class TestProportionalCapacityPreemptionPolicy {
when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT);
boolean preemptionDisabled = mockPreemptionStatus("root");
when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled);
+ QueueResourceQuotas rootQr = new QueueResourceQuotas();
+ rootQr.setEffectiveMaxResource(Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(abs[0]);
+ rootQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[0], 0));
+ rootQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[0]);
+ when(root.getQueueResourceQuotas()).thenReturn(rootQr);
+ when(root.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[0]);
+ when(root.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[0], 0));
for (int i = 1; i < queues.length; ++i) {
final CSQueue q;
@@ -1232,6 +1243,18 @@ public class TestProportionalCapacityPreemptionPolicy {
qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize());
when(q.getQueueCapacities()).thenReturn(qc);
+ QueueResourceQuotas qr = new QueueResourceQuotas();
+ qr.setEffectiveMaxResource(Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(abs[i]);
+ qr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(maxCap[i], 0));
+ qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]);
+ when(q.getQueueResourceQuotas()).thenReturn(qr);
+ when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(abs[i]);
+ when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(maxCap[i], 0));
+
String parentPathName = p.getQueuePath();
parentPathName = (parentPathName == null) ? "root" : parentPathName;
String queuePathName = (parentPathName + "." + queueName).replace("/",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
index 7784549..a1d89d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java
@@ -67,9 +67,9 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY,
"priority_first");
- String labelsConfig = "=100:200,true;";
+ String labelsConfig = "=100:50,true;";
String nodesConfig = // n1 has no label
- "n1= res=100:200";
+ "n1= res=100:50";
String queuesConfig =
// guaranteed,max,used,pending,reserved
"root(=[100:50 100:50 80:40 120:60 0]);" + // root
@@ -105,7 +105,7 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF
verify(mDisp, times(1)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(4))));
- verify(mDisp, times(7)).handle(argThat(
+ verify(mDisp, times(3)).handle(argThat(
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(3))));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
new file mode 100644
index 0000000..5a66281
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -0,0 +1,516 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestAbsoluteResourceConfiguration {
+
+ private static final int GB = 1024;
+
+ private static final String QUEUEA = "queueA";
+ private static final String QUEUEB = "queueB";
+ private static final String QUEUEC = "queueC";
+ private static final String QUEUEA1 = "queueA1";
+ private static final String QUEUEA2 = "queueA2";
+ private static final String QUEUEB1 = "queueB1";
+
+ private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEA;
+ private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEB;
+ private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT
+ + "." + QUEUEC;
+ private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1;
+ private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2;
+ private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1;
+
+ private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB,
+ 10);
+ private static final Resource QUEUE_A_MAXRES = Resource.newInstance(200 * GB,
+ 30);
+ private static final Resource QUEUE_A1_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_A2_MINRES = Resource.newInstance(50 * GB,
+ 5);
+ private static final Resource QUEUE_B_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_B1_MINRES = Resource.newInstance(40 * GB,
+ 10);
+ private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB,
+ 30);
+ private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB,
+ 10);
+ private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB,
+ 20);
+ private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6);
+ private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6);
+ private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000,
+ 20);
+
+ private static Set<String> resourceTypes = new HashSet<>(
+ Arrays.asList("memory", "vcores"));
+
+ private CapacitySchedulerConfiguration setupSimpleQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexQueueConfiguration(
+ boolean isCapacityNeeded) {
+ CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
+ csConf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[]{QUEUEA, QUEUEB, QUEUEC});
+ csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2});
+ csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1});
+
+ // Set default capacities like normal configuration.
+ if (isCapacityNeeded) {
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+ csConf.setCapacity(QUEUEA1_FULL, 50f);
+ csConf.setCapacity(QUEUEA2_FULL, 50f);
+ csConf.setCapacity(QUEUEB1_FULL, 100f);
+ }
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig(
+ CapacitySchedulerConfiguration csConf) {
+ // Update min/max resource to queueA/B/C
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES);
+ csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES);
+ csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES);
+
+ return csConf;
+ }
+
+ @Test
+ public void testSimpleMinMaxResourceConfigurartionPerQueue() {
+
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true);
+ setupMinMaxResourceConfiguration(csConf);
+
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes));
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES,
+ csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES,
+ csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes));
+ }
+
+ @Test
+ public void testEffectiveMinMaxResourceConfigurartionPerQueue()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+
+ @Test
+ public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases 1) Configure percentage based capacity and absolute
+ * resource together. 2) As per above tree structure, ensure all values
+ * could be retrieved. 3) Validate whether min resource cannot be more than
+ * max resources. 4) Validate whether max resource of queue cannot be more
+ * than its parent max resource.
+ */
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ // 1. Create a new config with capcity and min/max together. Ensure an
+ // exception is thrown.
+ CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
+ true);
+ setupMinMaxResourceConfiguration(csConf1);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Queue 'queueA' should use either"
+ + " percentage based capacity configuration or absolute resource.",
+ e.getMessage());
+ }
+ rm.stop();
+
+ // 2. Create a new config with min/max alone with a complex queue config.
+ // Check all values could be fetched correctly.
+ CapacitySchedulerConfiguration csConf2 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf2);
+
+ rm = new MockRM(csConf2);
+ rm.start();
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+ cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1);
+ Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct",
+ QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct",
+ QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2);
+ Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct",
+ QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct",
+ QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1);
+ Assert.assertNotNull(qB1);
+ Assert.assertEquals("Min resource configured for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct",
+ QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct",
+ QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // 3. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf3 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf3);
+
+ csConf3.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B_MAXRES);
+ csConf3.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES);
+
+ try {
+ cs.reinitialize(csConf3, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Min resource configuration "
+ + "<memory:153600, vCores:30> is greater than its "
+ + "max value:<memory:40960, vCores:10> in queue:queueB1",
+ e.getMessage());
+ }
+
+ // 4. Create a new config and make sure one queue's max resource is more
+ // than its preant's max resource configured.
+ CapacitySchedulerConfiguration csConf4 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf4);
+
+ csConf4.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_A_MAXRES);
+
+ try {
+ cs.reinitialize(csConf4, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert
+ .assertEquals(
+ "Failed to re-init queues : Max resource configuration "
+ + "<memory:204800, vCores:30> is greater than parents max value:"
+ + "<memory:153600, vCores:30> in queue:queueB1",
+ e.getMessage());
+ }
+ rm.stop();
+ }
+
+ @Test
+ public void testComplexValidateAbsoluteResourceConfig() throws Exception {
+ /**
+ * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ *
+ * Test below cases: 1) Parent and its child queues must use either
+ * percentage based or absolute resource configuration. 2) Parent's min
+ * resource must be more than sum of child's min resource.
+ */
+
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf);
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ rm.registerNode("127.0.0.1:1234", 250 * GB, 40);
+
+ // 1. Explicitly set percentage based config for parent queues. This will
+ // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute
+ // resource.
+ csConf.setCapacity(QUEUEA_FULL, 50f);
+ csConf.setCapacity(QUEUEB_FULL, 25f);
+ csConf.setCapacity(QUEUEC_FULL, 25f);
+
+ // Also unset resource based config.
+ csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
+ csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ try {
+ cs.reinitialize(csConf, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals(
+ "Failed to re-init queues : Parent queue 'queueA' "
+ + "and child queue 'queueA1' should use either percentage based"
+ + " capacity configuration or absolute resource together.",
+ e.getMessage());
+ }
+
+ // 2. Create a new config and make sure one queue's min resource is more
+ // than its max resource configured.
+ CapacitySchedulerConfiguration csConf1 = setupComplexQueueConfiguration(
+ false);
+ setupComplexMinMaxResourceConfig(csConf1);
+
+ // Configure QueueA with lesser resource than its children.
+ csConf1.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A1_MINRES);
+
+ try {
+ cs.reinitialize(csConf1, rm.getRMContext());
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e instanceof IOException);
+ Assert.assertEquals("Failed to re-init queues : Parent Queues capacity: "
+ + "<memory:51200, vCores:5> is less than to its children:"
+ + "<memory:102400, vCores:10> for queue:queueA", e.getMessage());
+ }
+ }
+
+ @Test
+ public void testEffectiveResourceAfterReducingClusterResource()
+ throws Exception {
+ // create conf with basic queue configuration.
+ CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(
+ false);
+ setupMinMaxResourceConfiguration(csConf);
+
+ csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ @SuppressWarnings("resource")
+ MockRM rm = new MockRM(csConf);
+ rm.start();
+
+ // Add few nodes
+ MockNM nm1 = rm.registerNode("127.0.0.1:1234", 125 * GB, 20);
+ rm.registerNode("127.0.0.2:1234", 125 * GB, 20);
+
+ // Get queue object to verify min/max resource configuration.
+ CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+
+ LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA);
+ Assert.assertNotNull(qA);
+ Assert.assertEquals("Min resource configured for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB);
+ Assert.assertNotNull(qB);
+ Assert.assertEquals("Min resource configured for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC);
+ Assert.assertNotNull(qC);
+ Assert.assertEquals("Min resource configured for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource());
+ Assert.assertEquals("Max resource configured for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource());
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ // unregister one NM.
+ rm.unRegisterNode(nm1);
+
+ // After loosing one NM, effective min res of queueA will become just
+ // above half. Hence A's min will be 60Gi and 6 cores and max will be
+ // 128GB and 20 cores.
+ Assert.assertEquals("Effective Min resource for QUEUEA is not correct",
+ QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEA is not correct",
+ QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEB is not correct",
+ QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEB is not correct",
+ QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource());
+
+ Assert.assertEquals("Effective Min resource for QUEUEC is not correct",
+ QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource());
+ Assert.assertEquals("Effective Max resource for QUEUEC is not correct",
+ QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource());
+
+ rm.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 8aca235..24ae244 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -86,6 +87,7 @@ public class TestApplicationLimits {
final static int GB = 1024;
LeafQueue queue;
+ CSQueue root;
private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -100,7 +102,7 @@ public class TestApplicationLimits {
setupQueueConfiguration(csConf);
rmContext = TestUtils.getMockRMContext();
-
+ Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32);
CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class);
when(csContext.getConfiguration()).thenReturn(csConf);
@@ -110,10 +112,11 @@ public class TestApplicationLimits {
when(csContext.getMaximumResourceCapability()).
thenReturn(Resources.createResource(16*GB, 32));
when(csContext.getClusterResource()).
- thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32));
+ thenReturn(clusterResource);
when(csContext.getResourceCalculator()).
thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
RMContainerTokenSecretManager containerTokenSecretManager =
new RMContainerTokenSecretManager(conf);
@@ -122,13 +125,17 @@ public class TestApplicationLimits {
containerTokenSecretManager);
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
- CSQueue root = CapacitySchedulerQueueManager
+ root = CapacitySchedulerQueueManager
.parseQueue(csContext, csConf, null, "root",
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
-
queue = spy(new LeafQueue(csContext, A, root, null));
+ QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A))
+ .getQueueResourceQuotas();
+ doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas();
// Stub out ACL checks
doReturn(true).
@@ -189,6 +196,8 @@ public class TestApplicationLimits {
// when there is only 1 user, and drops to 2G (the userlimit) when there
// is a second user
Resource clusterResource = Resource.newInstance(80 * GB, 40);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue.updateClusterResource(clusterResource, new ResourceLimits(
clusterResource));
@@ -287,6 +296,8 @@ public class TestApplicationLimits {
CSQueue root =
CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
"root", queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
LeafQueue queue = (LeafQueue)queues.get(A);
@@ -357,6 +368,8 @@ public class TestApplicationLimits {
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
clusterResource = Resources.createResource(100 * 16 * GB);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
@@ -378,6 +391,8 @@ public class TestApplicationLimits {
root = CapacitySchedulerQueueManager.parseQueue(
csContext, csConf, null, "root",
queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource, new ResourceLimits(
+ clusterResource));
queue = (LeafQueue)queues.get(A);
assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath()));
@@ -393,7 +408,7 @@ public class TestApplicationLimits {
final String user_0 = "user_0";
final String user_1 = "user_1";
final String user_2 = "user_2";
-
+
assertEquals(Resource.newInstance(16 * GB, 1),
queue.calculateAndGetAMResourceLimit());
assertEquals(Resource.newInstance(8 * GB, 1),
@@ -578,6 +593,7 @@ public class TestApplicationLimits {
thenReturn(Resources.createResource(16*GB));
when(csContext.getResourceCalculator()).thenReturn(resourceCalculator);
when(csContext.getRMContext()).thenReturn(rmContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
// Say cluster has 100 nodes of 16G each
Resource clusterResource = Resources.createResource(100 * 16 * GB);
@@ -586,6 +602,8 @@ public class TestApplicationLimits {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -693,6 +711,8 @@ public class TestApplicationLimits {
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.createResource(90*16*GB);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Any change is cluster resource needs to enforce user-limit recomputation.
// In existing code, LeafQueue#updateClusterResource handled this. However
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index 0aac2ef..d73f1c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -600,6 +601,7 @@ public class TestApplicationLimitsByPartition {
RMContext spyRMContext = spy(rmContext);
when(spyRMContext.getNodeLabelManager()).thenReturn(mgr);
when(csContext.getRMContext()).thenReturn(spyRMContext);
+ when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager());
mgr.activateNode(NodeId.newInstance("h0", 0),
Resource.newInstance(160 * GB, 16)); // default Label
@@ -615,6 +617,8 @@ public class TestApplicationLimitsByPartition {
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, "root", queues, queues, TestUtils.spyHook);
+ rootQueue.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 1edb0da..ebc5174 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -4306,7 +4306,7 @@ public class TestCapacityScheduler {
null, null, NULL_UPDATE_REQUESTS);
CapacityScheduler.schedule(cs);
}
- assertEquals("P2 Used Resource should be 8 GB", 8 * GB,
+ assertEquals("P2 Used Resource should be 7 GB", 7 * GB,
cs.getQueue("p2").getUsedResources().getMemorySize());
//Free a container from X1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index 0fcc86d..fcfa0dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -242,6 +242,8 @@ public class TestChildQueueOrder {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index a32352b..96f8970 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -191,6 +191,8 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration.ROOT,
queues, queues,
TestUtils.spyHook);
+ root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32),
+ new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32)));
ResourceUsage queueResUsage = root.getQueueResourceUsage();
when(csContext.getClusterResourceUsage())
@@ -307,13 +309,11 @@ public class TestLeafQueue {
// Verify the value for getAMResourceLimit for queues with < .1 maxcap
Resource clusterResource = Resource.newInstance(50 * GB, 50);
- a.updateClusterResource(clusterResource,
+ root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
assertEquals(Resource.newInstance(1 * GB, 1),
a.calculateAndGetAMResourceLimit());
- b.updateClusterResource(clusterResource,
- new ResourceLimits(clusterResource));
assertEquals(Resource.newInstance(5 * GB, 1),
b.calculateAndGetAMResourceLimit());
}
@@ -358,6 +358,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -556,6 +558,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -630,6 +634,8 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -699,6 +705,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (80 * GB), numNodes * 100);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Set user-limit. Need a small queue within a large cluster.
b.setUserLimit(50);
@@ -779,6 +787,8 @@ public class TestLeafQueue {
Resources.createResource(numNodes * (8 * GB), numNodes * 100);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getClusterResource()).thenReturn(clusterResource);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests so that one application is memory dominant
// and other application is vcores dominant
@@ -891,6 +901,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -915,6 +927,8 @@ public class TestLeafQueue {
// Set user-limit
a.setUserLimit(50);
a.setUserLimitFactor(2);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// There're two active users
assertEquals(2, a.getAbstractUsersManager().getNumActiveUsers());
@@ -940,7 +954,7 @@ public class TestLeafQueue {
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
// Allocate one container to app_0, before allocating this container,
- // user-limit = ceil((4 + 1) / 2) = 3G. app_0's used resource (3G) <=
+ // user-limit = floor((5 + 1) / 2) = 3G. app_0's used resource (3G) <=
// user-limit.
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_1,
@@ -1068,15 +1082,9 @@ public class TestLeafQueue {
a.assignContainers(clusterResource, node_1,
new ResourceLimits(clusterResource),
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(9*GB, a.getUsedResources().getMemorySize());
- assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
-
- assertEquals(4*GB,
- app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize());
-
- assertEquals(1*GB,
- app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize());
+ assertEquals(12*GB, a.getUsedResources().getMemorySize());
+ assertEquals(12*GB, app_0.getCurrentConsumption().getMemorySize());
+ assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
}
@SuppressWarnings({ "unchecked", "rawtypes" })
@@ -1100,6 +1108,8 @@ public class TestLeafQueue {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
CapacitySchedulerQueueManager mockCapacitySchedulerQueueManager
= mock(CapacitySchedulerQueueManager.class);
@@ -1122,6 +1132,8 @@ public class TestLeafQueue {
qb.setUserLimit(100);
qb.setUserLimitFactor(1);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
final ApplicationAttemptId appAttemptId_0 =
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
@@ -1256,106 +1268,6 @@ public class TestLeafQueue {
}
@Test
- public void testUserHeadroomMultiApp() throws Exception {
- // Mock the queue
- LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
- //unset maxCapacity
- a.setMaxCapacity(1.0f);
-
- // Users
- final String user_0 = "user_0";
- final String user_1 = "user_1";
-
- // Submit applications
- final ApplicationAttemptId appAttemptId_0 =
- TestUtils.getMockApplicationAttemptId(0, 0);
- FiCaSchedulerApp app_0 =
- new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_0, user_0);
-
- final ApplicationAttemptId appAttemptId_1 =
- TestUtils.getMockApplicationAttemptId(1, 0);
- FiCaSchedulerApp app_1 =
- new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_1, user_0); // same user
-
- final ApplicationAttemptId appAttemptId_2 =
- TestUtils.getMockApplicationAttemptId(2, 0);
- FiCaSchedulerApp app_2 =
- new FiCaSchedulerApp(appAttemptId_2, user_1, a,
- a.getAbstractUsersManager(), spyRMContext);
- a.submitApplicationAttempt(app_2, user_1);
-
- // Setup some nodes
- String host_0 = "127.0.0.1";
- FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK,
- 0, 16*GB);
- String host_1 = "127.0.0.2";
- FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK,
- 0, 16*GB);
-
- Map<ApplicationAttemptId, FiCaSchedulerApp> apps = ImmutableMap.of(
- app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(),
- app_1, app_2.getApplicationAttemptId(), app_2);
- Map<NodeId, FiCaSchedulerNode> nodes = ImmutableMap.of(node_0.getNodeID(),
- node_0, node_1.getNodeID(), node_1);
-
- final int numNodes = 2;
- Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1);
- when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
- Priority priority = TestUtils.createMockPriority(1);
-
- app_0.updateResourceRequests(Collections.singletonList(
- TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
- priority, recordFactory)));
-
- applyCSAssignment(clusterResource,
- a.assignContainers(clusterResource, node_0,
- new ResourceLimits(clusterResource),
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(1*GB, a.getUsedResources().getMemorySize());
- assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
- //Now, headroom is the same for all apps for a given user + queue combo
- //and a change to any app's headroom is reflected for all the user's apps
- //once those apps are active/have themselves calculated headroom for
- //allocation at least one time
- assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize());//not yet active
- assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
- app_1.updateResourceRequests(Collections.singletonList(
- TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
- priority, recordFactory)));
-
- applyCSAssignment(clusterResource,
- a.assignContainers(clusterResource, node_0,
- new ResourceLimits(clusterResource),
- SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps);
- assertEquals(2*GB, a.getUsedResources().getMemorySize());
- assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(1*GB, app_1.getHeadroom().getMemorySize());//now active
- assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active
-
- //Complete container and verify that headroom is updated, for both apps
- //for the user
- RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
- a.completedContainer(clusterResource, app_0, node_0, rmContainer,
- ContainerStatus.newInstance(rmContainer.getContainerId(),
- ContainerState.COMPLETE, "",
- ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
- RMContainerEventType.KILL, null, true);
-
- assertEquals(2*GB, app_0.getHeadroom().getMemorySize());
- assertEquals(2*GB, app_1.getHeadroom().getMemorySize());
- }
-
- @Test
public void testHeadroomWithMaxCap() throws Exception {
// Mock the queue
LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
@@ -1403,7 +1315,12 @@ public class TestLeafQueue {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
-
+
+ ParentQueue root = (ParentQueue) queues
+ .get(CapacitySchedulerConfiguration.ROOT);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
@@ -1454,6 +1371,8 @@ public class TestLeafQueue {
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
priority, recordFactory)));
@@ -1542,6 +1461,8 @@ public class TestLeafQueue {
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getClusterResource()).thenReturn(clusterResource);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -1624,6 +1545,8 @@ public class TestLeafQueue {
// Test max-capacity
// Now - no more allocs since we are at max-cap
a.setMaxCapacity(0.5f);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -1638,6 +1561,8 @@ public class TestLeafQueue {
// Now, allocations should goto app_3 since it's under user-limit
a.setMaxCapacity(1.0f);
a.setUserLimitFactor(1);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
applyCSAssignment(clusterResource,
a.assignContainers(clusterResource, node_0,
new ResourceLimits(clusterResource),
@@ -1743,6 +1668,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (4*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priority = TestUtils.createMockPriority(1);
@@ -1880,6 +1807,8 @@ public class TestLeafQueue {
final int numNodes = 3;
Resource clusterResource =
Resources.createResource(numNodes * (4*GB), numNodes * 16);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
when(csContext.getMaximumResourceCapability()).thenReturn(
Resources.createResource(4*GB, 16));
@@ -2051,6 +1980,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2237,11 +2168,10 @@ public class TestLeafQueue {
CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
TestUtils.spyHook);
- queues = newQueues;
root.reinitialize(newRoot, cs.getClusterResource());
// Manipulate queue 'b'
- LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));
+ LeafQueue a = stubLeafQueue((LeafQueue) newQueues.get(B));
// Check locality parameters.
assertEquals(2, a.getNodeLocalityDelay());
@@ -2277,6 +2207,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8 * GB), numNodes * 16);
when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes);
+ newRoot.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2412,6 +2344,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
List<ResourceRequest> app_0_requests_0 = new ArrayList<ResourceRequest>();
@@ -2545,6 +2479,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(
numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
Priority priority = TestUtils.createMockPriority(1);
@@ -2660,17 +2596,14 @@ public class TestLeafQueue {
assertEquals(2, e.getNumActiveApplications());
assertEquals(1, e.getNumPendingApplications());
- csConf.setDouble(CapacitySchedulerConfiguration
- .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
- CapacitySchedulerConfiguration
- .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2);
+ csConf.setDouble(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT
+ * 2);
Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
- CSQueue newRoot =
- CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
- CapacitySchedulerConfiguration.ROOT,
- newQueues, queues,
- TestUtils.spyHook);
- queues = newQueues;
+ CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+ csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+ TestUtils.spyHook);
root.reinitialize(newRoot, csContext.getClusterResource());
// after reinitialization
@@ -2697,7 +2630,6 @@ public class TestLeafQueue {
CapacitySchedulerConfiguration.ROOT,
newQueues, queues,
TestUtils.spyHook);
- queues = newQueues;
root.reinitialize(newRoot, cs.getClusterResource());
// after reinitialization
@@ -2745,7 +2677,7 @@ public class TestLeafQueue {
assertEquals(1, e.getNumPendingApplications());
Resource clusterResource = Resources.createResource(200 * 16 * GB, 100 * 32);
- e.updateClusterResource(clusterResource,
+ root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
// after updating cluster resource
@@ -2837,6 +2769,9 @@ public class TestLeafQueue {
numNodes * (8*GB), numNodes * 1);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
// resourceName: <priority, memory, #containers, relaxLocality>
// host_0_0: < 1, 1GB, 1, true >
@@ -3036,36 +2971,44 @@ public class TestLeafQueue {
@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
throws Exception {
+ Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
- Resource clusterResource = Resources
- .createResource(100 * 16 * GB, 100 * 32);
+ final String newRootName = "root" + System.currentTimeMillis();
+ setupQueueConfiguration(csConf, newRootName);
+
+ Resource clusterResource = Resources.createResource(100 * 16 * GB,
+ 100 * 32);
CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
when(csContext.getRMContext()).thenReturn(rmContext);
- csConf.setFloat(CapacitySchedulerConfiguration.
- MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
- ParentQueue root = new ParentQueue(csContext,
- CapacitySchedulerConfiguration.ROOT, null, null);
- csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
- LeafQueue a = new LeafQueue(csContext, A, root, null);
- assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(160 * GB, 1));
-
- csConf.setFloat(CapacitySchedulerConfiguration.
- MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
- LeafQueue newA = new LeafQueue(csContext, A, root, null);
- a.reinitialize(newA, clusterResource);
- assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(320 * GB, 1));
+ csConf.setFloat(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ 0.1f);
+
+ CSQueue root;
+ root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
- Resource newClusterResource = Resources.createResource(100 * 20 * GB,
- 100 * 32);
- a.updateClusterResource(newClusterResource,
- new ResourceLimits(newClusterResource));
- // 100 * 20 * 0.2 = 400
- assertEquals(a.calculateAndGetAMResourceLimit(),
- Resources.createResource(400 * GB, 1));
+ // Manipulate queue 'a'
+ LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B));
+ assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f);
+ assertEquals(b.calculateAndGetAMResourceLimit(),
+ Resources.createResource(159 * GB, 1));
+
+ csConf.setFloat(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ 0.2f);
+ clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32);
+ Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+ CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext,
+ csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
+ TestUtils.spyHook);
+ root.reinitialize(newRoot, clusterResource);
+
+ b = stubLeafQueue((LeafQueue) newQueues.get(B));
+ assertEquals(b.calculateAndGetAMResourceLimit(),
+ Resources.createResource(320 * GB, 1));
}
@Test
@@ -3142,6 +3085,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(numNodes * (16 * GB),
numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
String user_0 = "user_0";
@@ -3308,6 +3253,8 @@ public class TestLeafQueue {
Resource clusterResource = Resources.createResource(
numNodes * (16*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
String user_0 = "user_0";
@@ -3435,6 +3382,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (8*GB), numNodes * 16);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests and submit
// App0 has node local request for host_0/host_1, and app1 has node local
@@ -3533,6 +3482,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (100*GB), numNodes * 128);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Pending resource requests for app_0 and app_1 total 5GB.
Priority priority = TestUtils.createMockPriority(1);
@@ -3699,6 +3650,8 @@ public class TestLeafQueue {
Resource clusterResource =
Resources.createResource(numNodes * (100*GB), numNodes * 128);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Pending resource requests for user_0: app_0 and app_1 total 3GB.
Priority priority = TestUtils.createMockPriority(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index a9196d1..25a9774 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -249,6 +249,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
@@ -494,6 +496,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
CSQueue a = queues.get(A);
@@ -710,6 +714,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue a = (LeafQueue)queues.get(A);
@@ -790,6 +796,8 @@ public class TestParentQueue {
Resources.createResource(numNodes * (memoryPerNode*GB),
numNodes * coresPerNode);
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Start testing
LeafQueue b3 = (LeafQueue)queues.get(B3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf4a43e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 32f022f..0855720 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -267,6 +267,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -454,6 +456,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -600,6 +604,8 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -782,6 +788,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -896,6 +904,10 @@ public class TestReservations {
String host_1 = "host_1";
FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
8 * GB);
+
+ Resource clusterResource = Resources.createResource(2 * 8 * GB);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
@@ -1068,6 +1080,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1256,6 +1270,8 @@ public class TestReservations {
final int numNodes = 2;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
@@ -1418,6 +1434,9 @@ public class TestReservations {
final int numNodes = 3;
Resource clusterResource = Resources.createResource(numNodes * (8 * GB));
when(csContext.getNumClusterNodes()).thenReturn(numNodes);
+ root.updateClusterResource(clusterResource,
+ new ResourceLimits(clusterResource));
+
// Setup resource-requests
Priority priorityAM = TestUtils.createMockPriority(1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/49] hadoop git commit: YARN-7330. Add support to show GPU in UI
including metrics. Contributed by Wangda Tan.
Posted by su...@apache.org.
YARN-7330. Add support to show GPU in UI including metrics. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aab43959
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aab43959
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aab43959
Branch: refs/heads/YARN-5881
Commit: aab439593b5d3459140b6e104722d4583dfcfddb
Parents: 4cc9479
Author: Sunil G <su...@apache.org>
Authored: Thu Nov 23 07:54:20 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Nov 23 07:54:20 2017 +0530
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 8 +
.../hadoop/yarn/api/records/Resource.java | 21 +++
.../resources/gpu/GpuResourceAllocator.java | 19 +-
.../resources/gpu/GpuResourceHandlerImpl.java | 1 -
.../resourceplugin/ResourcePlugin.java | 11 ++
.../resourceplugin/gpu/AssignedGpuDevice.java | 79 ++++++++
.../resourceplugin/gpu/GpuDevice.java | 4 +-
.../resourceplugin/gpu/GpuResourcePlugin.java | 24 ++-
.../nodemanager/webapp/NMWebServices.java | 28 +++
.../nodemanager/webapp/dao/NMResourceInfo.java | 28 +++
.../webapp/dao/gpu/GpuDeviceInformation.java | 2 +-
.../webapp/dao/gpu/NMGpuResourceInfo.java | 71 +++++++
.../webapp/dao/gpu/PerGpuDeviceInformation.java | 2 +-
.../webapp/dao/gpu/PerGpuMemoryUsage.java | 2 +-
.../resources/gpu/TestGpuResourceHandler.java | 6 +-
.../nodemanager/webapp/TestNMWebServices.java | 185 +++++++++++++++----
.../dao/gpu/TestGpuDeviceInformationParser.java | 2 +-
.../src/main/webapp/app/adapters/yarn-nm-gpu.js | 33 ++++
.../main/webapp/app/components/donut-chart.js | 18 +-
.../webapp/app/components/gpu-donut-chart.js | 66 +++++++
.../src/main/webapp/app/constants.js | 13 ++
.../webapp/app/controllers/yarn-nodes/table.js | 2 +-
.../main/webapp/app/models/cluster-metric.js | 69 +++++++
.../src/main/webapp/app/models/yarn-nm-gpu.js | 27 +++
.../app/models/yarn-queue/capacity-queue.js | 3 +-
.../src/main/webapp/app/models/yarn-rm-node.js | 35 ++++
.../src/main/webapp/app/router.js | 5 +-
.../main/webapp/app/routes/cluster-overview.js | 2 +-
.../src/main/webapp/app/routes/yarn-node.js | 2 +
.../webapp/app/routes/yarn-node/yarn-nm-gpu.js | 22 +++
.../main/webapp/app/serializers/yarn-nm-gpu.js | 43 +++++
.../serializers/yarn-queue/capacity-queue.js | 1 +
.../main/webapp/app/serializers/yarn-rm-node.js | 4 +-
.../webapp/app/templates/cluster-overview.hbs | 88 ++++++---
.../templates/components/node-menu-panel.hbs | 10 +-
.../templates/components/yarn-nm-gpu-info.hbs | 69 +++++++
.../src/main/webapp/app/templates/yarn-node.hbs | 125 -------------
.../webapp/app/templates/yarn-node/info.hbs | 154 +++++++++++++++
.../app/templates/yarn-node/yarn-nm-gpu.hbs | 53 ++++++
.../src/main/webapp/app/utils/converter.js | 51 +++++
40 files changed, 1181 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 670b7ee..de4b0e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -643,4 +643,12 @@
<Method name="serveNIOTCP" />
<Bug pattern="RV_RETURN_VALUE_IGNORED_BAD_PRACTICE" />
</Match>
+
+ <!-- EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice" />
+ <Method name="equals" />
+ <Bug pattern="EQ_OVERRIDING_EQUALS_NOT_SYMMETRIC" />
+ </Match>
+
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 65b5dce..abd44b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -18,9 +18,14 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.NotImplementedException;
+import org.apache.curator.shaded.com.google.common.reflect.ClassPath;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -209,6 +214,22 @@ public abstract class Resource implements Comparable<Resource> {
}
/**
+ * Get list of resource information, this will be used by JAXB.
+ * @return list of resources copy.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public List<ResourceInformation> getAllResourcesListCopy() {
+ List<ResourceInformation> list = new ArrayList<>();
+ for (ResourceInformation i : resources) {
+ ResourceInformation ri = new ResourceInformation();
+ ResourceInformation.copy(i, ri);
+ list.add(ri);
+ }
+ return list;
+ }
+
+ /**
* Get ResourceInformation for a specified resource.
*
* @param resource name of the resource
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
index f2bb342..5bdffc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceAllocator.java
@@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -224,7 +226,20 @@ public class GpuResourceAllocator {
}
@VisibleForTesting
- public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMapping() {
- return new HashMap<>(usedDevices);
+ public synchronized Map<GpuDevice, ContainerId> getDeviceAllocationMappingCopy() {
+ return new HashMap<>(usedDevices);
+ }
+
+ public synchronized List<GpuDevice> getAllowedGpusCopy() {
+ return new ArrayList<>(allowedGpuDevices);
+ }
+
+ public synchronized List<AssignedGpuDevice> getAssignedGpusCopy() {
+ List<AssignedGpuDevice> assigns = new ArrayList<>();
+ for (Map.Entry<GpuDevice, ContainerId> entry : usedDevices.entrySet()) {
+ assigns.add(new AssignedGpuDevice(entry.getKey().getIndex(),
+ entry.getKey().getMinorNumber(), entry.getValue()));
+ }
+ return assigns;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
index 2dbd3e3..8ddc227 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java
@@ -153,7 +153,6 @@ public class GpuResourceHandlerImpl implements ResourceHandler {
return null;
}
- @VisibleForTesting
public GpuResourceAllocator getGpuAllocator() {
return gpuAllocator;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
index 99a18ed..f141179 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/ResourcePlugin.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerChain;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
/**
* {@link ResourcePlugin} is an interface for node manager to easier support
@@ -91,4 +92,14 @@ public interface ResourcePlugin {
* have requirement to update docker command.
*/
DockerCommandPlugin getDockerCommandPluginInstance();
+
+ /**
+ * Get resource information from this plugin.
+ *
+ * @return NMResourceInfo, an example is
+ * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation}
+ *
+ * @throws YarnException when any issue occurs
+ */
+ NMResourceInfo getNMResourceInfo() throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
new file mode 100644
index 0000000..26fd905
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/AssignedGpuDevice.java
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * In addition to {@link GpuDevice}, this include container id and more runtime
+ * information related to who is using the GPU device if possible
+ */
+public class AssignedGpuDevice extends GpuDevice {
+ private static final long serialVersionUID = -12983712986315L;
+
+ String containerId;
+
+ public AssignedGpuDevice(int index, int minorNumber,
+ ContainerId containerId) {
+ super(index, minorNumber);
+ this.containerId = containerId.toString();
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public void setContainerId(String containerId) {
+ this.containerId = containerId;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == null || !(obj instanceof AssignedGpuDevice)) {
+ return false;
+ }
+ AssignedGpuDevice other = (AssignedGpuDevice) obj;
+ return index == other.index && minorNumber == other.minorNumber
+ && containerId.equals(other.containerId);
+ }
+
+ @Override
+ public int compareTo(Object obj) {
+ if (obj == null || (!(obj instanceof AssignedGpuDevice))) {
+ return -1;
+ }
+
+ AssignedGpuDevice other = (AssignedGpuDevice) obj;
+
+ int result = Integer.compare(index, other.index);
+ if (0 != result) {
+ return result;
+ }
+ result = Integer.compare(minorNumber, other.minorNumber);
+ if (0 != result) {
+ return result;
+ }
+ return containerId.compareTo(other.containerId);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 47;
+ return prime * (prime * index + minorNumber) + containerId.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
index 8119924..bce1d9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuDevice.java
@@ -24,8 +24,8 @@ import java.io.Serializable;
* This class is used to represent GPU device while allocation.
*/
public class GpuDevice implements Serializable, Comparable {
- private int index;
- private int minorNumber;
+ protected int index;
+ protected int minorNumber;
private static final long serialVersionUID = -6812314470754667710L;
public GpuDevice(int index, int minorNumber) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
index 4ff186f..f28218d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/gpu/GpuResourcePlugin.java
@@ -18,18 +18,26 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.CGroupsHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceAllocator;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.gpu.GpuResourceHandlerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.DockerCommandPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.NodeResourceUpdaterPlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo;
+
+import java.util.List;
+import java.util.Map;
public class GpuResourcePlugin implements ResourcePlugin {
- private ResourceHandler gpuResourceHandler = null;
+ private GpuResourceHandlerImpl gpuResourceHandler = null;
private GpuNodeResourceUpdateHandler resourceDiscoverHandler = null;
private DockerCommandPlugin dockerCommandPlugin = null;
@@ -67,4 +75,18 @@ public class GpuResourcePlugin implements ResourcePlugin {
public DockerCommandPlugin getDockerCommandPluginInstance() {
return dockerCommandPlugin;
}
+
+ @Override
+ public NMResourceInfo getNMResourceInfo() throws YarnException {
+ GpuDeviceInformation gpuDeviceInformation =
+ GpuDiscoverer.getInstance().getGpuDeviceInformation();
+ GpuResourceAllocator gpuResourceAllocator =
+ gpuResourceHandler.getGpuAllocator();
+ List<GpuDevice> totalGpus = gpuResourceAllocator.getAllowedGpusCopy();
+ List<AssignedGpuDevice> assignedGpuDevices =
+ gpuResourceAllocator.getAssignedGpusCopy();
+
+ return new NMGpuResourceInfo(gpuDeviceInformation, totalGpus,
+ assignedGpuDevices);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index c5379cc..937a8cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -27,6 +27,10 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -496,6 +500,30 @@ public class NMWebServices {
}
}
+ @GET
+ @Path("/resources/{resourcename}")
+ @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
+ public Object getNMResourceInfo(
+ @PathParam("resourcename")
+ String resourceName) throws YarnException {
+ init();
+ ResourcePluginManager rpm = this.nmContext.getResourcePluginManager();
+ if (rpm != null && rpm.getNameToPlugins() != null) {
+ ResourcePlugin plugin = rpm.getNameToPlugins().get(resourceName);
+ if (plugin != null) {
+ NMResourceInfo nmResourceInfo = plugin.getNMResourceInfo();
+ if (nmResourceInfo != null) {
+ return nmResourceInfo;
+ }
+ }
+ }
+
+ throw new YarnException(
+ "Could not get detailed resource information for given resource-name="
+ + resourceName);
+ }
+
private long parseLongParam(String bytes) {
if (bytes == null || bytes.isEmpty()) {
return Long.MAX_VALUE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java
new file mode 100644
index 0000000..18ce8ea
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NMResourceInfo.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NMResourceInfo {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
index 977032a..837d5cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/GpuDeviceInformation.java
@@ -25,7 +25,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.List;
/**
- * All GPU Device Information in the system.
+ * All GPU Device Information in the system, fetched from nvidia-smi.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java
new file mode 100644
index 0000000..bf1d463
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/NMGpuResourceInfo.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+
+import java.util.List;
+
+/**
+ * Gpu device information return to client when
+ * {@link org.apache.hadoop.yarn.server.nodemanager.webapp.NMWebServices#getNMResourceInfo(String)}
+ * is invoked.
+ */
+public class NMGpuResourceInfo extends NMResourceInfo {
+ GpuDeviceInformation gpuDeviceInformation;
+
+ List<GpuDevice> totalGpuDevices;
+ List<AssignedGpuDevice> assignedGpuDevices;
+
+ public NMGpuResourceInfo(GpuDeviceInformation gpuDeviceInformation,
+ List<GpuDevice> totalGpuDevices,
+ List<AssignedGpuDevice> assignedGpuDevices) {
+ this.gpuDeviceInformation = gpuDeviceInformation;
+ this.totalGpuDevices = totalGpuDevices;
+ this.assignedGpuDevices = assignedGpuDevices;
+ }
+
+ public GpuDeviceInformation getGpuDeviceInformation() {
+ return gpuDeviceInformation;
+ }
+
+ public void setGpuDeviceInformation(
+ GpuDeviceInformation gpuDeviceInformation) {
+ this.gpuDeviceInformation = gpuDeviceInformation;
+ }
+
+ public List<GpuDevice> getTotalGpuDevices() {
+ return totalGpuDevices;
+ }
+
+ public void setTotalGpuDevices(List<GpuDevice> totalGpuDevices) {
+ this.totalGpuDevices = totalGpuDevices;
+ }
+
+ public List<AssignedGpuDevice> getAssignedGpuDevices() {
+ return assignedGpuDevices;
+ }
+
+ public void setAssignedGpuDevices(
+ List<AssignedGpuDevice> assignedGpuDevices) {
+ this.assignedGpuDevices = assignedGpuDevices;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
index f315313..25c2e3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuDeviceInformation.java
@@ -135,7 +135,7 @@ public class PerGpuDeviceInformation {
this.gpuUtilizations = utilizations;
}
- @XmlElement(name = "bar1_memory_usage")
+ @XmlElement(name = "fb_memory_usage")
public PerGpuMemoryUsage getGpuMemoryUsage() {
return gpuMemoryUsage;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
index 3964c4e..afc1a96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/PerGpuMemoryUsage.java
@@ -27,7 +27,7 @@ import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
@InterfaceAudience.Private
@InterfaceStability.Unstable
-@XmlRootElement(name = "bar1_memory_usage")
+@XmlRootElement(name = "fb_memory_usage")
public class PerGpuMemoryUsage {
long usedMemoryMiB = -1L;
long availMemoryMiB = -1L;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
index 1e0eb7b..87ab249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java
@@ -368,7 +368,7 @@ public class TestGpuResourceHandler {
gpuResourceHandler.reacquireContainer(getContainerId(1));
Map<GpuDevice, ContainerId> deviceAllocationMapping =
- gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(
deviceAllocationMapping.keySet().contains(new GpuDevice(1, 1)));
@@ -402,7 +402,7 @@ public class TestGpuResourceHandler {
// Make sure internal state not changed.
deviceAllocationMapping =
- gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(deviceAllocationMapping.keySet()
.containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
@@ -434,7 +434,7 @@ public class TestGpuResourceHandler {
// Make sure internal state not changed.
deviceAllocationMapping =
- gpuResourceHandler.getGpuAllocator().getDeviceAllocationMapping();
+ gpuResourceHandler.getGpuAllocator().getDeviceAllocationMappingCopy();
Assert.assertEquals(2, deviceAllocationMapping.size());
Assert.assertTrue(deviceAllocationMapping.keySet()
.containsAll(Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 3))));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index fbab34a..171834e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -18,31 +18,20 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.io.StringReader;
-import java.net.HttpURLConnection;
-import java.net.URI;
-import java.net.URL;
-import java.util.List;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.MediaType;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
-import org.apache.hadoop.http.JettyUtils;
-import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
+import com.google.inject.Guice;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
+import com.sun.jersey.api.client.UniformInterfaceException;
+import com.sun.jersey.api.client.WebResource;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.WebAppDescriptor;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.http.JettyUtils;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -50,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.logaggregation.ContainerLogAggregationType;
import org.apache.hadoop.yarn.logaggregation.ContainerLogFileInfo;
import org.apache.hadoop.yarn.logaggregation.TestContainerLogsUtils;
@@ -61,13 +51,22 @@ import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.AssignedGpuDevice;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.NMResourceInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.GpuDeviceInformation;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.NMGpuResourceInfo;
+import org.apache.hadoop.yarn.server.nodemanager.webapp.dao.gpu.PerGpuDeviceInformation;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webapp.YarnWebServiceParams;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerLogsInfo;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.GuiceServletConfig;
import org.apache.hadoop.yarn.webapp.JerseyTestBase;
import org.apache.hadoop.yarn.webapp.WebApp;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
@@ -83,22 +82,36 @@ import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-import com.google.inject.Guice;
-import com.google.inject.servlet.ServletModule;
-import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.ClientResponse.Status;
-import com.sun.jersey.api.client.GenericType;
-import com.sun.jersey.api.client.UniformInterfaceException;
-import com.sun.jersey.api.client.WebResource;
-import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
-import com.sun.jersey.test.framework.WebAppDescriptor;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.yarn.webapp.WebServicesTestUtils.assertResponseStatusCode;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Test the nodemanager node info web services api's
*/
public class TestNMWebServices extends JerseyTestBase {
- private static Context nmContext;
+ private static NodeManager.NMContext nmContext;
private static ResourceView resourceView;
private static ApplicationACLsManager aclsManager;
private static LocalDirsHandlerService dirsHandler;
@@ -418,6 +431,116 @@ public class TestNMWebServices extends JerseyTestBase {
assertFalse(redirectURL.contains(YarnWebServiceParams.NM_ID));
}
+ @Test
+ public void testGetNMResourceInfo()
+ throws YarnException, InterruptedException, JSONException {
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ Map<String, ResourcePlugin> namesToPlugins = new HashMap<>();
+ ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class);
+ NMResourceInfo nmResourceInfo1 = new NMResourceInfo() {
+ public long a = 1000L;
+ };
+ when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1);
+ namesToPlugins.put("resource-1", mockPlugin1);
+ namesToPlugins.put("yarn.io/resource-1", mockPlugin1);
+ ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class);
+ namesToPlugins.put("resource-2", mockPlugin2);
+ when(rpm.getNameToPlugins()).thenReturn(namesToPlugins);
+
+ nmContext.setResourcePluginManager(rpm);
+
+ WebResource r = resource();
+ ClientResponse response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-2").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+
+ // Access resource-2 should fail (null NMResourceInfo returned).
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertIncludesException(json);
+
+ // Access resource-3 should fail (unkown plugin)
+ response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-3").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ assertIncludesException(json);
+
+ // Access resource-1 should success
+ response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ Assert.assertEquals(1000, json.get("a"));
+
+ // Access resource-1 should success (encoded yarn.io/Fresource-1).
+ response = r.path("ws").path("v1").path("node").path("resources").path(
+ "yarn.io%2Fresource-1").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ Assert.assertEquals(1000, json.get("a"));
+ }
+
+ private ContainerId createContainerId(int id) {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId, id);
+ return containerId;
+ }
+
+ @Test
+ public void testGetYarnGpuResourceInfo()
+ throws YarnException, InterruptedException, JSONException {
+ ResourcePluginManager rpm = mock(ResourcePluginManager.class);
+ Map<String, ResourcePlugin> namesToPlugins = new HashMap<>();
+ ResourcePlugin mockPlugin1 = mock(ResourcePlugin.class);
+ GpuDeviceInformation gpuDeviceInformation = new GpuDeviceInformation();
+ gpuDeviceInformation.setDriverVersion("1.2.3");
+ gpuDeviceInformation.setGpus(Arrays.asList(new PerGpuDeviceInformation()));
+ NMResourceInfo nmResourceInfo1 = new NMGpuResourceInfo(gpuDeviceInformation,
+ Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2),
+ new GpuDevice(3, 3)), Arrays
+ .asList(new AssignedGpuDevice(2, 2, createContainerId(1)),
+ new AssignedGpuDevice(3, 3, createContainerId(2))));
+ when(mockPlugin1.getNMResourceInfo()).thenReturn(nmResourceInfo1);
+ namesToPlugins.put("resource-1", mockPlugin1);
+ namesToPlugins.put("yarn.io/resource-1", mockPlugin1);
+ ResourcePlugin mockPlugin2 = mock(ResourcePlugin.class);
+ namesToPlugins.put("resource-2", mockPlugin2);
+ when(rpm.getNameToPlugins()).thenReturn(namesToPlugins);
+
+ nmContext.setResourcePluginManager(rpm);
+
+ WebResource r = resource();
+ ClientResponse response;
+ JSONObject json;
+
+ // Access resource-1 should success
+ response = r.path("ws").path("v1").path("node").path(
+ "resources").path("resource-1").accept(MediaType.APPLICATION_JSON).get(
+ ClientResponse.class);
+ assertEquals(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
+ response.getType().toString());
+ json = response.getEntity(JSONObject.class);
+ Assert.assertEquals("1.2.3",
+ json.getJSONObject("gpuDeviceInformation").get("driverVersion"));
+ Assert.assertEquals(3, json.getJSONArray("totalGpuDevices").length());
+ Assert.assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
+ Assert.assertEquals(2, json.getJSONArray("assignedGpuDevices").length());
+ }
+
+ private void assertIncludesException(JSONObject json) {
+ Assert.assertTrue(json.has("RemoteException"));
+ }
+
private void testContainerLogs(WebResource r, ContainerId containerId)
throws IOException {
final String containerIdStr = containerId.toString();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
index e22597d..dc96746 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/gpu/TestGpuDeviceInformationParser.java
@@ -39,7 +39,7 @@ public class TestGpuDeviceInformationParser {
Assert.assertEquals(2, info.getGpus().size());
PerGpuDeviceInformation gpu1 = info.getGpus().get(1);
Assert.assertEquals("Tesla P100-PCIE-12GB", gpu1.getProductName());
- Assert.assertEquals(16384, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
+ Assert.assertEquals(12193, gpu1.getGpuMemoryUsage().getTotalMemoryMiB());
Assert.assertEquals(10.3f,
gpu1.getGpuUtilizations().getOverallGpuUtilization(), 1e-6);
Assert.assertEquals(34f, gpu1.getTemperature().getCurrentGpuTemp(), 1e-6);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js
new file mode 100644
index 0000000..bf6307a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-nm-gpu.js
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import AbstractAdapter from './abstract';
+
+export default AbstractAdapter.extend({
+
+ address: "localBaseAddress",
+ restNameSpace: "node",
+ serverName: "NM",
+
+ urlForFindRecord(id/*, modelName, snapshot*/) {
+ var url = this._buildURL();
+ url = url.replace("{nodeAddress}", id) + "/resources/yarn.io%2Fgpu";
+ return url;
+ }
+
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
index b1e6ecf..03b6336 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/donut-chart.js
@@ -20,6 +20,7 @@ import Ember from 'ember';
import BaseChartComponent from 'yarn-ui/components/base-chart-component';
import ColorUtils from 'yarn-ui/utils/color-utils';
import Converter from 'yarn-ui/utils/converter';
+import {Entities} from 'yarn-ui/constants';
export default BaseChartComponent.extend({
/*
@@ -41,8 +42,10 @@ export default BaseChartComponent.extend({
}
if (!middleValue) {
- if (this.get("type") === "memory") {
+ if (this.get(Entities.Type) === Entities.Memory) {
middleValue = Converter.memoryToSimpliedUnit(total);
+ } else if (this.get(Entities.Type) === Entities.Resource) {
+ middleValue = Converter.resourceToSimplifiedUnit(total, this.get(Entities.Unit));
} else {
middleValue = total;
}
@@ -151,7 +154,10 @@ export default BaseChartComponent.extend({
var value = d.value;
if (this.get("type") === "memory") {
value = Converter.memoryToSimpliedUnit(value);
+ } else if (this.get("type") === "resource") {
+ value = Converter.resourceToSimplifiedUnit(value, this.get(Entities.Unit));
}
+
return d.label + ' = ' + value + suffix;
}.bind(this));
}
@@ -185,10 +191,18 @@ export default BaseChartComponent.extend({
}
this.renderDonutChart(this.get("data"), this.get("title"), this.get("showLabels"),
- this.get("middleLabel"), this.get("middleValue"));
+ this.get("middleLabel"), this.get("middleValue"), this.get("suffix"));
},
didInsertElement: function() {
+ // When parentIdPrefix is specified, use parentidPrefix + name as new parent
+ // id
+ if (this.get("parentIdPrefix")) {
+ var newParentId = this.get("parentIdPrefix") + this.get("id");
+ this.set("parentId", newParentId);
+ console.log(newParentId);
+ }
+
this.initChart();
this.draw();
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js
new file mode 100644
index 0000000..fa5ca8a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/gpu-donut-chart.js
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DonutChart from 'yarn-ui/components/donut-chart';
+import ColorUtils from 'yarn-ui/utils/color-utils';
+
+export default DonutChart.extend({
+ draw: function() {
+ // Construct data
+ var data = [];
+ if (this.get("gpu-render-type") === "gpu-memory") {
+ data.push({
+ label: "Used",
+ value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.usedMemoryMiB),
+ });
+ data.push({
+ label: "Available",
+ value: parseFloat(this.get("gpuInfo").gpuMemoryUsage.availMemoryMiB)
+ });
+ } else if (this.get("gpu-render-type") === "gpu-utilization") {
+ var utilization = parseFloat(this.get("gpuInfo").gpuUtilizations.overallGpuUtilization);
+ data.push({
+ label: "Utilized",
+ value: utilization,
+ });
+ data.push({
+ label: "Available",
+ value: 100 - utilization
+ });
+ }
+
+ var colorTargets = this.get("colorTargets");
+ if (colorTargets) {
+ var colorTargetReverse = Boolean(this.get("colorTargetReverse"));
+ var targets = colorTargets.split(" ");
+ this.colors = ColorUtils.getColors(data.length, targets, colorTargetReverse);
+ }
+
+ this.renderDonutChart(data, this.get("title"), this.get("showLabels"),
+ this.get("middleLabel"), this.get("middleValue"), this.get("suffix"));
+ },
+
+ didInsertElement: function() {
+ // ParentId includes minorNumber
+ var newParentId = this.get("parentId") + this.get("gpuInfo").minorNumber;
+ this.set("parentId", newParentId);
+
+ this.initChart();
+ this.draw();
+ },
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
index d2937a0..29ad4bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/constants.js
@@ -22,3 +22,16 @@
export default {
PARAM_SEPARATOR: '!',
};
+
+const BASE_UNIT = 1024
+
+export const Type = 'type';
+export const Memory = 'memory';
+export const Resource = 'resource';
+export const Unit = 'unit';
+export const Entities = {
+ Type: 'type',
+ Memory:'memory',
+ Resource: 'resource',
+ Unit: 'unit'
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
index 30180dd..24e3878 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/controllers/yarn-nodes/table.js
@@ -67,7 +67,7 @@ export default Ember.Controller.extend({
getCellContent: function(row) {
var node_id = row.get("id"),
node_addr = row.get("nodeHTTPAddress"),
- href = `#/yarn-node/${node_id}/${node_addr}`;
+ href = `#/yarn-node/${node_id}/${node_addr}/info`;
switch(row.get("nodeState")) {
case "SHUTDOWN":
case "LOST":
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
index 0be0d83..bbc6f08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/cluster-metric.js
@@ -43,6 +43,8 @@ export default DS.Model.extend({
decommissionedNodes: DS.attr('number'),
rebootedNodes: DS.attr('number'),
activeNodes: DS.attr('number'),
+ totalUsedResourcesAcrossPartition: DS.attr('object'),
+ totalClusterResourcesAcrossPartition: DS.attr('object'),
getFinishedAppsDataForDonutChart: function() {
var arr = [];
@@ -135,4 +137,71 @@ export default DS.Model.extend({
return arr;
}.property("allocatedVirtualCores", "reservedVirtualCores", "availableVirtualCores"),
+
+ getResourceTypes: function() {
+ var types = [];
+ if (this.get("totalClusterResourcesAcrossPartition")) {
+
+ console.log(types);
+ }
+ }.property("totalClusterResourcesAcrossPartition"),
+
+ /*
+ * Returned format
+ * [
+ * {
+ * name: <resource-name>
+ * unit: <resource-unit>
+ * [
+ * {
+ * label: <label>
+ * value: <value>
+ * },
+ * {
+ * }
+ * ...
+ * ],
+ * }
+ * ]
+ */
+ getAllResourceTypesDonutChart: function() {
+ if (this.get("totalClusterResourcesAcrossPartition")
+ && this.get("totalUsedResourcesAcrossPartition")) {
+ var usages = [];
+
+ var clusterResourceInformations = this.get("totalClusterResourcesAcrossPartition").resourcesInformations;
+ var usedResourceInformations = this.get("totalUsedResourcesAcrossPartition").resourcesInformations;
+
+ clusterResourceInformations.forEach(function(cluster) {
+ var perResourceTypeUsage = {
+ name: cluster.name,
+ unit: cluster.units,
+ data: []
+ };
+
+ usedResourceInformations.forEach(function (used) {
+ if (used.name === perResourceTypeUsage.name) {
+ var usedValue = used.value;
+ perResourceTypeUsage.data.push({
+ label: "Used",
+ value: usedValue
+ }, {
+ label: "Available",
+ value: cluster.value - usedValue
+ });
+ }
+ });
+
+ usages.push(perResourceTypeUsage);
+
+ // Make sure id is a valid w3c ID
+ perResourceTypeUsage.id = perResourceTypeUsage.name.replace('/', '-');
+ perResourceTypeUsage.id = perResourceTypeUsage.id.replace('.', '-');
+ });
+
+ console.log(usages);
+ return usages;
+ }
+ return null;
+ }.property()
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js
new file mode 100644
index 0000000..b3e9c2a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-nm-gpu.js
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.Model.extend({
+ info: DS.attr('object'),
+
+ jsonString: function() {
+ return JSON.stringify(this.get("info"));
+ }.property(),
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
index b84a473..1d162e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
@@ -37,6 +37,7 @@ export default DS.Model.extend({
numActiveApplications: DS.attr('number'),
users: DS.hasMany('YarnUser'),
type: DS.attr('string'),
+ resources: DS.attr('object'),
isLeafQueue: function() {
var len = this.get("children.length");
@@ -93,5 +94,5 @@ export default DS.Model.extend({
value: this.get("numActiveApplications") || 0
}
];
- }.property("numPendingApplications", "numActiveApplications")
+ }.property("numPendingApplications", "numActiveApplications"),
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
index 1321074..da5ba01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-rm-node.js
@@ -32,6 +32,8 @@ export default DS.Model.extend({
availableVirtualCores: DS.attr('number'),
version: DS.attr('string'),
nodeLabels: DS.attr('array'),
+ availableResource: DS.attr('object'),
+ usedResource: DS.attr('object'),
nodeLabelsAsString: function() {
var labels = this.get("nodeLabels");
@@ -90,6 +92,39 @@ export default DS.Model.extend({
return arr;
}.property("availableVirtualCores", "usedVirtualCores"),
+ getGpuDataForDonutChart: function() {
+ var arr = [];
+ var used = 0;
+ var ri;
+
+ var resourceInformations = this.get("usedResource").resourcesInformations;
+ for (var i = 0; i < resourceInformations.length; i++) {
+ ri = resourceInformations[i];
+ if (ri.name === "yarn.io/gpu") {
+ used = ri.value;
+ }
+ }
+
+ var available = 0;
+ resourceInformations = this.get("availableResource").resourcesInformations;
+ for (i = 0; i < resourceInformations.length; i++) {
+ ri = resourceInformations[i];
+ if (ri.name === "yarn.io/gpu") {
+ available = ri.value;
+ }
+ }
+
+ arr.push({
+ label: "Used",
+ value: used
+ });
+ arr.push({
+ label: "Available",
+ value: available
+ });
+ return arr;
+ }.property("availableResource", "usedResource"),
+
toolTipText: function() {
return "<p>Rack: " + this.get("rack") + '</p>' +
"<p>Host: " + this.get("nodeHostName") + '</p>';
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
index 64e9ad2..27a3ac2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/router.js
@@ -38,7 +38,10 @@ Router.map(function() {
this.route('apps');
});
this.route('yarn-nodes-heatmap');
- this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' });
+ this.route('yarn-node', { path: '/yarn-node/:node_id/:node_addr' }, function() {
+ this.route("info");
+ this.route("yarn-nm-gpu");
+ });
this.route('yarn-node-apps', { path: '/yarn-node-apps/:node_id/:node_addr' });
this.route('yarn-node-app',
{ path: '/yarn-node-app/:node_id/:node_addr/:app_id' });
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
index d03ea0d..254ece4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
@@ -31,7 +31,7 @@ export default AbstractRoute.extend({
queues: this.store.query("yarn-queue.yarn-queue", {}).then((model) => {
let type = model.get('firstObject').get('type');
return this.store.query("yarn-queue." + type + "-queue", {});
- }),
+ })
});
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
index 3d54846..7ce615c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node.js
@@ -25,6 +25,7 @@ export default AbstractRoute.extend({
// Fetches data from both NM and RM. RM is queried to get node usage info.
return Ember.RSVP.hash({
nodeInfo: { id: param.node_id, addr: param.node_addr },
+ nmGpuInfo: this.store.findRecord('yarn-nm-gpu', param.node_addr, {reload:true}),
node: this.store.findRecord('yarn-node', param.node_addr, {reload: true}),
rmNode: this.store.findRecord('yarn-rm-node', param.node_id, {reload: true})
});
@@ -33,5 +34,6 @@ export default AbstractRoute.extend({
unloadAll() {
this.store.unloadAll('yarn-node');
this.store.unloadAll('yarn-rm-node');
+ this.store.unloadAll('yarn-nm-gpu');
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js
new file mode 100644
index 0000000..38ae5d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-node/yarn-nm-gpu.js
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Ember from 'ember';
+
+export default Ember.Route.extend({
+});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js
new file mode 100644
index 0000000..3567c68
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-nm-gpu.js
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.JSONAPISerializer.extend({
+ internalNormalizeSingleResponse(store, primaryModelClass, payload, id) {
+ if (payload.nodeInfo) {
+ payload = payload.nodeInfo;
+ }
+
+ var fixedPayload = {
+ id: id,
+ type: primaryModelClass.modelName,
+ attributes: {
+ info: payload
+ }
+ };
+ return fixedPayload;
+ },
+
+ normalizeSingleResponse(store, primaryModelClass, payload, id/*, requestType*/) {
+ // payload is of the form {"nodeInfo":{}}
+ var p = this.internalNormalizeSingleResponse(store,
+ primaryModelClass, payload, id);
+ return { data: p };
+ },
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index c7350ef..7626598 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -72,6 +72,7 @@ export default DS.JSONAPISerializer.extend({
preemptionDisabled: payload.preemptionDisabled,
numPendingApplications: payload.numPendingApplications,
numActiveApplications: payload.numActiveApplications,
+ resources: payload.resources,
type: "capacity",
},
// Relationships
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
index 1c6d1be..a3a1d59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-rm-node.js
@@ -41,7 +41,9 @@ export default DS.JSONAPISerializer.extend({
usedVirtualCores: payload.usedVirtualCores,
availableVirtualCores: payload.availableVirtualCores,
version: payload.version,
- nodeLabels: payload.nodeLabels
+ nodeLabels: payload.nodeLabels,
+ usedResource: payload.used,
+ availableResource: payload.avail
}
};
return fixedPayload;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
index e549ce5..ff4682a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/cluster-overview.hbs
@@ -90,41 +90,71 @@
<hr>
<div class="row">
-
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - Memory
+ <!-- When getAllResourceTypesDonutChart is not null, use it to show per-resource-type usages. Otherwise only show
+ vcore/memory usage from metrics -->
+ {{#if model.clusterMetrics.firstObject.getAllResourceTypesDonutChart}}
+ {{#each
+ model.clusterMetrics.firstObject.getAllResourceTypesDonutChart as |perTypeUsage|}}
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ {{perTypeUsage.name}} - Usages
+ </div>
+ <div class="container-fluid" id="resource-type-{{perTypeUsage.id}}">
+ {{donut-chart
+ data=perTypeUsage.data
+ showLabels=true
+ parentIdPrefix="resource-type-"
+ id=perTypeUsage.id
+ ratio=0.6
+ unit=perTypeUsage.unit
+ type="resource"
+ maxHeight=350
+ colorTargets="good"
+ colorTargetReverse=true}}
+ </div>
+ </div>
</div>
- <div class="container-fluid" id="mem-donut-chart">
- {{donut-chart data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart
- showLabels=true
- parentId="mem-donut-chart"
- ratio=0.6
- maxHeight=350
- colorTargets="good"
- colorTargetReverse=true
- type="memory"}}
+ {{/each}}
+ {{else}}
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - Memory
+ </div>
+ <div class="container-fluid" id="mem-donut-chart">
+ {{donut-chart
+ data=model.clusterMetrics.firstObject.getMemoryDataForDonutChart
+ showLabels=true
+ parentId="mem-donut-chart"
+ ratio=0.6
+ maxHeight=350
+ colorTargets="good"
+ colorTargetReverse=true
+ type="memory"}}
+ </div>
</div>
</div>
- </div>
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Resource - VCores
- </div>
- <div class="container-fluid" id="vcore-donut-chart">
- {{donut-chart data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart
- showLabels=true
- parentId="vcore-donut-chart"
- ratio=0.6
- maxHeight=350
- colorTargets="good"
- colorTargetReverse=true}}
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Resource - VCores
+ </div>
+ <div class="container-fluid" id="vcore-donut-chart">
+ {{donut-chart
+ data=model.clusterMetrics.firstObject.getVCoreDataForDonutChart
+ showLabels=true
+ parentId="vcore-donut-chart"
+ ratio=0.6
+ maxHeight=350
+ colorTargets="good"
+ colorTargetReverse=true}}
+ </div>
</div>
</div>
- </div>
+ {{/if}}
+
</div>
<div class="row">
<div class="col-lg-6 container-fluid">
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
index d2486c9..fffae30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/node-menu-panel.hbs
@@ -24,8 +24,8 @@
<div class="panel-body">
<ul class="nav nav-pills nav-stacked" id="stacked-menu">
<ul class="nav nav-pills nav-stacked collapse in">
- {{#link-to 'yarn-node' tagName="li"}}
- {{#link-to 'yarn-node' nodeId nodeAddr}}Node Information
+ {{#link-to 'yarn-node.info' tagName="li"}}
+ {{#link-to 'yarn-node.info' nodeId nodeAddr}}Node Information
{{/link-to}}
{{/link-to}}
{{#link-to 'yarn-node-apps' tagName="li"}}
@@ -36,6 +36,12 @@
{{#link-to 'yarn-node-containers' nodeId nodeAddr}}List of Containers
{{/link-to}}
{{/link-to}}
+ {{#if nmGpuInfo}}
+ {{#link-to 'yarn-node.yarn-nm-gpu' tagName="li"}}
+ {{#link-to 'yarn-node.yarn-nm-gpu' nodeId nodeAddr }}GPU Information
+ {{/link-to}}
+ {{/link-to}}
+ {{/if}}
</ul>
</ul>
</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab43959/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs
new file mode 100644
index 0000000..4118b1e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-nm-gpu-info.hbs
@@ -0,0 +1,69 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<div class="panel panel-default">
+ <div class="panel-heading">Gpu Information - (Minor
+ Number {{gpu.minorNumber}})
+ </div>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Product Name</td>
+ <td>{{gpu.productName}}</td>
+ </tr>
+ <tr>
+ <td>UUID</td>
+ <td>{{gpu.uuid}}</td>
+ </tr>
+ <tr>
+ <td>Current Temperature</td>
+ <td>{{gpu.temperature.currentGpuTemp}}</td>
+ </tr>
+ <tr>
+ <td>Max Temperature</td>
+ <td>{{gpu.temperature.maxGpuTemp}}</td>
+ </tr>
+ </tbody>
+ </table>
+
+ <div class="col-md-5 container-fluid" id="mem-donut-chart{{gpu.minorNumber}}">
+ {{gpu-donut-chart gpuInfo=gpu
+ showLabels=true
+ parentId="mem-donut-chart"
+ middleLabel = "Gpu Memory"
+ ratio=0.6
+ type="memory"
+ gpu-render-type = "gpu-memory"
+ colorTargets="good"
+ colorTargetReverse=true
+ maxHeight=350}}
+ </div>
+
+ <div class="col-md-5 container-fluid" id="utilization-donut-chart{{gpu.minorNumber}}">
+ {{gpu-donut-chart gpuInfo=gpu
+ showLabels=true
+ parentId="utilization-donut-chart"
+ middleLabel = "Gpu Utilization"
+ ratio=0.6
+ gpu-render-type = "gpu-utilization"
+ colorTargets="good"
+ colorTargetReverse=true
+ suffix="%"
+ maxHeight=350}}
+ </div>
+</div>
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/49] hadoop git commit: HDFS-12754. Lease renewal can hit a
deadlock. Contributed by Kuhu Shukla.
Posted by su...@apache.org.
HDFS-12754. Lease renewal can hit a deadlock. Contributed by Kuhu Shukla.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/738d1a20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/738d1a20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/738d1a20
Branch: refs/heads/YARN-5881
Commit: 738d1a206aba05f0b4be7d633b17db7fcd1c74bc
Parents: 785732c
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Nov 22 13:01:07 2017 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Nov 22 13:01:55 2017 -0600
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 15 +++-
.../hadoop/hdfs/DFSClientFaultInjector.java | 2 +
.../hadoop/hdfs/client/impl/LeaseRenewer.java | 55 +++++---------
.../hdfs/client/impl/TestLeaseRenewer.java | 23 ++----
.../hadoop/hdfs/TestDFSClientRetries.java | 80 ++++++++++++++++++++
.../hadoop/hdfs/TestDistributedFileSystem.java | 5 +-
6 files changed, 122 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/738d1a20/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 467d6be..5652321 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -483,12 +483,21 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
/** Get a lease and start automatic renewal */
private void beginFileLease(final long inodeId, final DFSOutputStream out)
throws IOException {
- getLeaseRenewer().put(inodeId, out, this);
+ synchronized (filesBeingWritten) {
+ putFileBeingWritten(inodeId, out);
+ getLeaseRenewer().put(this);
+ }
}
/** Stop renewal of lease for the file. */
void endFileLease(final long inodeId) {
- getLeaseRenewer().closeFile(inodeId, this);
+ synchronized (filesBeingWritten) {
+ removeFileBeingWritten(inodeId);
+ // remove client from renewer if no files are open
+ if (filesBeingWritten.isEmpty()) {
+ getLeaseRenewer().closeClient(this);
+ }
+ }
}
@@ -614,9 +623,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
@Override
public synchronized void close() throws IOException {
if(clientRunning) {
+ // lease renewal stops when all files are closed
closeAllFilesBeingWritten(false);
clientRunning = false;
- getLeaseRenewer().closeClient(this);
// close connections to the namenode
closeConnectionToNamenode();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/738d1a20/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
index b58cf16..d36c058 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
@@ -63,4 +63,6 @@ public class DFSClientFaultInjector {
}
public void sleepBeforeHedgedGet() {}
+
+ public void delayWhenRenewLeaseTimeout() {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/738d1a20/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
index 6faf133..e33d024 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
@@ -30,7 +30,7 @@ import java.util.Map;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSOutputStream;
+import org.apache.hadoop.hdfs.DFSClientFaultInjector;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Daemon;
@@ -76,7 +76,7 @@ import org.slf4j.LoggerFactory;
public class LeaseRenewer {
static final Logger LOG = LoggerFactory.getLogger(LeaseRenewer.class);
- static final long LEASE_RENEWER_GRACE_DEFAULT = 60*1000L;
+ private static long leaseRenewerGraceDefault = 60*1000L;
static final long LEASE_RENEWER_SLEEP_DEFAULT = 1000L;
/** Get a {@link LeaseRenewer} instance */
@@ -156,9 +156,7 @@ public class LeaseRenewer {
final LeaseRenewer stored = renewers.get(r.factorykey);
//Since a renewer may expire, the stored renewer can be different.
if (r == stored) {
- if (!r.clientsRunning()) {
- renewers.remove(r.factorykey);
- }
+ renewers.remove(r.factorykey);
}
}
}
@@ -201,7 +199,7 @@ public class LeaseRenewer {
private LeaseRenewer(Factory.Key factorykey) {
this.factorykey = factorykey;
- unsyncSetGraceSleepPeriod(LEASE_RENEWER_GRACE_DEFAULT);
+ unsyncSetGraceSleepPeriod(leaseRenewerGraceDefault);
if (LOG.isTraceEnabled()) {
instantiationTrace = StringUtils.stringifyException(
@@ -293,8 +291,7 @@ public class LeaseRenewer {
&& Time.monotonicNow() - emptyTime > gracePeriod;
}
- public synchronized void put(final long inodeId, final DFSOutputStream out,
- final DFSClient dfsc) {
+ public synchronized void put(final DFSClient dfsc) {
if (dfsc.isClientRunning()) {
if (!isRunning() || isRenewerExpired()) {
//start a new deamon with a new id.
@@ -328,7 +325,6 @@ public class LeaseRenewer {
});
daemon.start();
}
- dfsc.putFileBeingWritten(inodeId, out);
emptyTime = Long.MAX_VALUE;
}
}
@@ -338,28 +334,6 @@ public class LeaseRenewer {
emptyTime = time;
}
- /** Close a file. */
- public void closeFile(final long inodeId, final DFSClient dfsc) {
- dfsc.removeFileBeingWritten(inodeId);
-
- synchronized(this) {
- if (dfsc.isFilesBeingWrittenEmpty()) {
- dfsclients.remove(dfsc);
- }
- //update emptyTime if necessary
- if (emptyTime == Long.MAX_VALUE) {
- for(DFSClient c : dfsclients) {
- if (!c.isFilesBeingWrittenEmpty()) {
- //found a non-empty file-being-written map
- return;
- }
- }
- //discover the first time that all file-being-written maps are empty.
- emptyTime = Time.monotonicNow();
- }
- }
- }
-
/** Close the given client. */
public synchronized void closeClient(final DFSClient dfsc) {
dfsclients.remove(dfsc);
@@ -447,14 +421,17 @@ public class LeaseRenewer {
} catch (SocketTimeoutException ie) {
LOG.warn("Failed to renew lease for " + clientsString() + " for "
+ (elapsed/1000) + " seconds. Aborting ...", ie);
+ List<DFSClient> dfsclientsCopy;
synchronized (this) {
- while (!dfsclients.isEmpty()) {
- DFSClient dfsClient = dfsclients.get(0);
- dfsClient.closeAllFilesBeingWritten(true);
- closeClient(dfsClient);
- }
+ DFSClientFaultInjector.get().delayWhenRenewLeaseTimeout();
+ dfsclientsCopy = new ArrayList<>(dfsclients);
+ dfsclients.clear();
//Expire the current LeaseRenewer thread.
emptyTime = 0;
+ Factory.INSTANCE.remove(LeaseRenewer.this);
+ }
+ for (DFSClient dfsClient : dfsclientsCopy) {
+ dfsClient.closeAllFilesBeingWritten(true);
}
break;
} catch (IOException ie) {
@@ -511,4 +488,10 @@ public class LeaseRenewer {
return b.append("]").toString();
}
}
+
+ @VisibleForTesting
+ public static void setLeaseRenewerGraceDefault(
+ long leaseRenewerGraceDefault) {
+ LeaseRenewer.leaseRenewerGraceDefault = leaseRenewerGraceDefault;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/738d1a20/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java
index eb10e96..f73ea6d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/client/impl/TestLeaseRenewer.java
@@ -109,7 +109,7 @@ public class TestLeaseRenewer {
// Set up a file so that we start renewing our lease.
DFSOutputStream mockStream = Mockito.mock(DFSOutputStream.class);
long fileId = 123L;
- renewer.put(fileId, mockStream, MOCK_DFSCLIENT);
+ renewer.put(MOCK_DFSCLIENT);
// Wait for lease to get renewed
long failTime = Time.monotonicNow() + 5000;
@@ -121,7 +121,7 @@ public class TestLeaseRenewer {
Assert.fail("Did not renew lease at all!");
}
- renewer.closeFile(fileId, MOCK_DFSCLIENT);
+ renewer.closeClient(MOCK_DFSCLIENT);
}
/**
@@ -136,11 +136,8 @@ public class TestLeaseRenewer {
Mockito.doReturn(false).when(mockClient1).renewLease();
assertSame(renewer, LeaseRenewer.getInstance(
FAKE_AUTHORITY, FAKE_UGI_A, mockClient1));
-
- // Set up a file so that we start renewing our lease.
- DFSOutputStream mockStream1 = Mockito.mock(DFSOutputStream.class);
long fileId = 456L;
- renewer.put(fileId, mockStream1, mockClient1);
+ renewer.put(mockClient1);
// Second DFSClient does renew lease
final DFSClient mockClient2 = createMockClient();
@@ -148,9 +145,7 @@ public class TestLeaseRenewer {
assertSame(renewer, LeaseRenewer.getInstance(
FAKE_AUTHORITY, FAKE_UGI_A, mockClient2));
- // Set up a file so that we start renewing our lease.
- DFSOutputStream mockStream2 = Mockito.mock(DFSOutputStream.class);
- renewer.put(fileId, mockStream2, mockClient2);
+ renewer.put(mockClient2);
// Wait for lease to get renewed
@@ -171,19 +166,17 @@ public class TestLeaseRenewer {
}
}, 100, 10000);
- renewer.closeFile(fileId, mockClient1);
- renewer.closeFile(fileId, mockClient2);
+ renewer.closeClient(mockClient1);
+ renewer.closeClient(mockClient2);
}
@Test
public void testThreadName() throws Exception {
- DFSOutputStream mockStream = Mockito.mock(DFSOutputStream.class);
- long fileId = 789L;
Assert.assertFalse("Renewer not initially running",
renewer.isRunning());
// Pretend to open a file
- renewer.put(fileId, mockStream, MOCK_DFSCLIENT);
+ renewer.put(MOCK_DFSCLIENT);
Assert.assertTrue("Renewer should have started running",
renewer.isRunning());
@@ -193,7 +186,7 @@ public class TestLeaseRenewer {
Assert.assertEquals("LeaseRenewer:myuser@hdfs://nn1/", threadName);
// Pretend to close the file
- renewer.closeFile(fileId, MOCK_DFSCLIENT);
+ renewer.closeClient(MOCK_DFSCLIENT);
renewer.setEmptyTime(Time.monotonicNow());
// Should stop the renewer running within a few seconds
http://git-wip-us.apache.org/repos/asf/hadoop/blob/738d1a20/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
index 9674454..880e3df 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSClientRetries.java
@@ -47,6 +47,7 @@ import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
@@ -1253,4 +1254,83 @@ public class TestDFSClientRetries {
cluster.shutdown();
}
}
+
+ @Test(timeout=120000)
+ public void testLeaseRenewAndDFSOutputStreamDeadLock() throws Exception {
+ CountDownLatch testLatch = new CountDownLatch(1);
+ DFSClientFaultInjector.set(new DFSClientFaultInjector() {
+ public void delayWhenRenewLeaseTimeout() {
+ try {
+ testLatch.await();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ String file1 = "/testFile1";
+ // Set short retry timeouts so this test runs faster
+ conf.setInt(DFS_CLIENT_SOCKET_TIMEOUT_KEY, 1000);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).build();
+ try {
+ cluster.waitActive();
+ final NamenodeProtocols spyNN = spy(cluster.getNameNodeRpc());
+
+ doAnswer(new SleepFixedTimeAnswer(1500, testLatch)).when(spyNN).complete(
+ anyString(), anyString(), any(ExtendedBlock.class), anyLong());
+ DFSClient client = new DFSClient(null, spyNN, conf, null);
+ // Get hold of the lease renewer instance used by the client
+ LeaseRenewer leaseRenewer = client.getLeaseRenewer();
+ leaseRenewer.setRenewalTime(100);
+ final OutputStream out1 = client.create(file1, false);
+
+ out1.write(new byte[256]);
+
+ Thread closeThread = new Thread(new Runnable() {
+ @Override public void run() {
+ try {
+ //1. trigger get LeaseRenewer lock
+ Mockito.doThrow(new SocketTimeoutException()).when(spyNN)
+ .renewLease(Mockito.anyString());
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ closeThread.start();
+
+ //2. trigger get DFSOutputStream lock
+ out1.close();
+
+ } finally {
+ cluster.shutdown();
+ }
+ }
+
+ private static class SleepFixedTimeAnswer implements Answer<Object> {
+ private final int sleepTime;
+ private final CountDownLatch testLatch;
+
+ SleepFixedTimeAnswer(int sleepTime, CountDownLatch latch) {
+ this.sleepTime = sleepTime;
+ this.testLatch = latch;
+ }
+
+ @Override
+ public Object answer(InvocationOnMock invocation) throws Throwable {
+ boolean interrupted = false;
+ try {
+ Thread.sleep(sleepTime);
+ } catch (InterruptedException ie) {
+ interrupted = true;
+ }
+ try {
+ return invocation.callRealMethod();
+ } finally {
+ testLatch.countDown();
+ if (interrupted) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/738d1a20/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 6389f0d..823c747 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -424,6 +424,7 @@ public class TestDistributedFileSystem {
Configuration conf = getTestConfiguration();
final long grace = 1000L;
MiniDFSCluster cluster = null;
+ LeaseRenewer.setLeaseRenewerGraceDefault(grace);
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(2).build();
@@ -436,10 +437,6 @@ public class TestDistributedFileSystem {
{
final DistributedFileSystem dfs = cluster.getFileSystem();
- Method setMethod = dfs.dfs.getLeaseRenewer().getClass()
- .getDeclaredMethod("setGraceSleepPeriod", long.class);
- setMethod.setAccessible(true);
- setMethod.invoke(dfs.dfs.getLeaseRenewer(), grace);
Method checkMethod = dfs.dfs.getLeaseRenewer().getClass()
.getDeclaredMethod("isRunning");
checkMethod.setAccessible(true);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/49] hadoop git commit: MAPREDUCE-7011.
TestClientDistributedCacheManager::testDetermineCacheVisibilities assumes all
parent dirs set other exec
Posted by su...@apache.org.
MAPREDUCE-7011. TestClientDistributedCacheManager::testDetermineCacheVisibilities assumes all parent dirs set other exec
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/782ba3bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/782ba3bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/782ba3bf
Branch: refs/heads/YARN-5881
Commit: 782ba3bf9da52699b27405a3f147464975d1df99
Parents: 03c311e
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Nov 21 20:42:28 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Nov 21 20:42:28 2017 -0800
----------------------------------------------------------------------
.../TestClientDistributedCacheManager.java | 28 +++++++++++++-------
1 file changed, 18 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/782ba3bf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
index b427f39..a61e938 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/filecache/TestClientDistributedCacheManager.java
@@ -34,12 +34,15 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
+
import org.junit.After;
-import org.junit.Assert;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import org.junit.Before;
import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -97,15 +100,15 @@ public class TestClientDistributedCacheManager {
FileStatus firstStatus = statCache.get(firstCacheFile.toUri());
FileStatus secondStatus = statCache.get(secondCacheFile.toUri());
- Assert.assertNotNull(firstCacheFile + " was not found in the stats cache",
+ assertNotNull(firstCacheFile + " was not found in the stats cache",
firstStatus);
- Assert.assertNotNull(secondCacheFile + " was not found in the stats cache",
+ assertNotNull(secondCacheFile + " was not found in the stats cache",
secondStatus);
- Assert.assertEquals("Missing/extra entries found in the stas cache",
+ assertEquals("Missing/extra entries found in the stats cache",
2, statCache.size());
String expected = firstStatus.getModificationTime() + ","
+ secondStatus.getModificationTime();
- Assert.assertEquals(expected, jobConf.get(MRJobConfig.CACHE_FILE_TIMESTAMPS));
+ assertEquals(expected, jobConf.get(MRJobConfig.CACHE_FILE_TIMESTAMPS));
job = Job.getInstance(conf);
job.addCacheFile(new Path(TEST_VISIBILITY_CHILD_DIR, "*").toUri());
@@ -115,12 +118,12 @@ public class TestClientDistributedCacheManager {
FileStatus thirdStatus = statCache.get(TEST_VISIBILITY_CHILD_DIR.toUri());
- Assert.assertEquals("Missing/extra entries found in the stas cache",
+ assertEquals("Missing/extra entries found in the stats cache",
1, statCache.size());
- Assert.assertNotNull(TEST_VISIBILITY_CHILD_DIR
+ assertNotNull(TEST_VISIBILITY_CHILD_DIR
+ " was not found in the stats cache", thirdStatus);
expected = Long.toString(thirdStatus.getModificationTime());
- Assert.assertEquals("Incorrect timestamp for " + TEST_VISIBILITY_CHILD_DIR,
+ assertEquals("Incorrect timestamp for " + TEST_VISIBILITY_CHILD_DIR,
expected, jobConf.get(MRJobConfig.CACHE_FILE_TIMESTAMPS));
}
@@ -141,6 +144,11 @@ public class TestClientDistributedCacheManager {
job.addCacheFile(relativePath.toUri());
jobConf = job.getConfiguration();
+ // skip test if scratch dir is not PUBLIC
+ assumeTrue(TEST_VISIBILITY_PARENT_DIR + " is not public",
+ ClientDistributedCacheManager.isPublic(
+ jobConf, TEST_VISIBILITY_PARENT_DIR.toUri(), statCache));
+
ClientDistributedCacheManager.determineCacheVisibilities(jobConf,
statCache);
// We use get() instead of getBoolean() so we can tell the difference
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/49] hadoop git commit: YARN-6128. Add support for AMRMProxy HA.
(Botong Huang via Subru).
Posted by su...@apache.org.
YARN-6128. Add support for AMRMProxy HA. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5f66888
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5f66888
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5f66888
Branch: refs/heads/YARN-5881
Commit: d5f66888b8d767ee6706fab9950c194a1bf26d32
Parents: 0940e4f
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Nov 17 17:39:06 2017 -0800
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Nov 17 17:39:06 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 13 +
.../src/main/resources/yarn-default.xml | 21 ++
.../hadoop-yarn-server-common/pom.xml | 5 +
.../utils/FederationRegistryClient.java | 338 +++++++++++++++++++
.../yarn/server/uam/UnmanagedAMPoolManager.java | 141 ++++++--
.../server/uam/UnmanagedApplicationManager.java | 212 +++++++-----
.../yarn/server/utils/AMRMClientUtils.java | 30 +-
.../yarn/server/MockResourceManagerFacade.java | 103 +++---
.../utils/TestFederationRegistryClient.java | 90 +++++
.../uam/TestUnmanagedApplicationManager.java | 100 +++++-
.../amrmproxy/AMRMProxyApplicationContext.java | 16 +
.../AMRMProxyApplicationContextImpl.java | 35 +-
.../nodemanager/amrmproxy/AMRMProxyService.java | 83 ++++-
.../amrmproxy/FederationInterceptor.java | 221 +++++++++++-
.../containermanager/ContainerManagerImpl.java | 9 +-
.../amrmproxy/BaseAMRMProxyTest.java | 14 +-
.../amrmproxy/TestAMRMProxyService.java | 21 +-
.../amrmproxy/TestFederationInterceptor.java | 126 ++++++-
.../TestableFederationInterceptor.java | 29 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 6 +-
.../src/site/markdown/Federation.md | 11 +-
21 files changed, 1345 insertions(+), 279 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 34257ed..ead9977 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2096,6 +2096,9 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE =
"org.apache.hadoop.yarn.server.nodemanager.amrmproxy."
+ "DefaultRequestInterceptor";
+ public static final String AMRM_PROXY_HA_ENABLED = NM_PREFIX
+ + "amrmproxy.ha.enable";
+ public static final boolean DEFAULT_AMRM_PROXY_HA_ENABLED = false;
/**
* Default platform-agnostic CLASSPATH for YARN applications. A
@@ -2930,6 +2933,11 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
FEDERATION_PREFIX + "cache-ttl.secs";
+ public static final String FEDERATION_REGISTRY_BASE_KEY =
+ FEDERATION_PREFIX + "registry.base-dir";
+ public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
+ "yarnfederation/";
+
// 5 minutes
public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
@@ -3087,6 +3095,11 @@ public class YarnConfiguration extends Configuration {
// Other Configs
////////////////////////////////
+ public static final String YARN_REGISTRY_CLASS =
+ YARN_PREFIX + "registry.class";
+ public static final String DEFAULT_YARN_REGISTRY_CLASS =
+ "org.apache.hadoop.registry.client.impl.FSRegistryOperationsService";
+
/**
* Use YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS instead.
* The interval of the yarn client's querying application state after
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e90d0f2..12cb902 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2826,7 +2826,20 @@
<value>300</value>
</property>
+ <property>
+ <description>The registry base directory for federation.</description>
+ <name>yarn.federation.registry.base-dir</name>
+ <value>yarnfederation/</value>
+ </property>
+
<!-- Other Configuration -->
+
+ <property>
+ <description>The registry implementation to use.</description>
+ <name>yarn.registry.class</name>
+ <value>org.apache.hadoop.registry.client.impl.FSRegistryOperationsService</value>
+ </property>
+
<property>
<description>The interval that the yarn client library uses to poll the
completion status of the asynchronous API of application client protocol.
@@ -2989,6 +3002,14 @@
<property>
<description>
+ Whether AMRMProxy HA is enabled.
+ </description>
+ <name>yarn.nodemanager.amrmproxy.ha.enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
Setting that controls whether distributed scheduling is enabled.
</description>
<name>yarn.nodemanager.distributed-scheduling.enabled</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 43ae3af..cd5195d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -67,6 +67,11 @@
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-registry</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java
new file mode 100644
index 0000000..6624318
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/utils/FederationRegistryClient.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import java.io.IOException;
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.registry.client.api.BindFlags;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Helper class that handles reads and writes to Yarn Registry to support UAM HA
+ * and second attempt.
+ */
+public class FederationRegistryClient {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(FederationRegistryClient.class);
+
+ private RegistryOperations registry;
+
+ private UserGroupInformation user;
+
+ // AppId -> SubClusterId -> UAM token
+ private Map<ApplicationId, Map<String, Token<AMRMTokenIdentifier>>>
+ appSubClusterTokenMap;
+
+ // Structure in registry: <registryBaseDir>/<AppId>/<SubClusterId> -> UAMToken
+ private String registryBaseDir;
+
+ public FederationRegistryClient(Configuration conf,
+ RegistryOperations registry, UserGroupInformation user) {
+ this.registry = registry;
+ this.user = user;
+ this.appSubClusterTokenMap = new ConcurrentHashMap<>();
+ this.registryBaseDir =
+ conf.get(YarnConfiguration.FEDERATION_REGISTRY_BASE_KEY,
+ YarnConfiguration.DEFAULT_FEDERATION_REGISTRY_BASE_KEY);
+ LOG.info("Using registry {} with base directory: {}",
+ this.registry.getClass().getName(), this.registryBaseDir);
+ }
+
+ /**
+ * Get the list of known applications in the registry.
+ *
+ * @return the list of known applications
+ */
+ public List<String> getAllApplications() {
+ // Suppress the exception here because it is valid that the entry does not
+ // exist
+ List<String> applications = null;
+ try {
+ applications = listDirRegistry(this.registry, this.user,
+ getRegistryKey(null, null), false);
+ } catch (YarnException e) {
+ LOG.warn("Unexpected exception from listDirRegistry", e);
+ }
+ if (applications == null) {
+ // It is valid for listDirRegistry to return null
+ return new ArrayList<>();
+ }
+ return applications;
+ }
+
+ /**
+ * For testing, delete all application records in registry.
+ */
+ @VisibleForTesting
+ public void cleanAllApplications() {
+ try {
+ removeKeyRegistry(this.registry, this.user, getRegistryKey(null, null),
+ true, false);
+ } catch (YarnException e) {
+ LOG.warn("Unexpected exception from removeKeyRegistry", e);
+ }
+ }
+
+ /**
+ * Write/update the UAM token for an application and a sub-cluster.
+ *
+ * @param subClusterId sub-cluster id of the token
+ * @param token the UAM of the application
+ * @return whether the amrmToken is added or updated to a new value
+ */
+ public boolean writeAMRMTokenForUAM(ApplicationId appId,
+ String subClusterId, Token<AMRMTokenIdentifier> token) {
+ Map<String, Token<AMRMTokenIdentifier>> subClusterTokenMap =
+ this.appSubClusterTokenMap.get(appId);
+ if (subClusterTokenMap == null) {
+ subClusterTokenMap = new ConcurrentHashMap<>();
+ this.appSubClusterTokenMap.put(appId, subClusterTokenMap);
+ }
+
+ boolean update = !token.equals(subClusterTokenMap.get(subClusterId));
+ if (!update) {
+ LOG.debug("Same amrmToken received from {}, skip writing registry for {}",
+ subClusterId, appId);
+ return update;
+ }
+
+ LOG.info("Writing/Updating amrmToken for {} to registry for {}",
+ subClusterId, appId);
+ try {
+ // First, write the token entry
+ writeRegistry(this.registry, this.user,
+ getRegistryKey(appId, subClusterId), token.encodeToUrlString(), true);
+
+ // Then update the subClusterTokenMap
+ subClusterTokenMap.put(subClusterId, token);
+ } catch (YarnException | IOException e) {
+ LOG.error(
+ "Failed writing AMRMToken to registry for subcluster " + subClusterId,
+ e);
+ }
+ return update;
+ }
+
+ /**
+ * Load the information of one application from registry.
+ *
+ * @param appId application id
+ * @return the sub-cluster to UAM token mapping
+ */
+ public Map<String, Token<AMRMTokenIdentifier>>
+ loadStateFromRegistry(ApplicationId appId) {
+ Map<String, Token<AMRMTokenIdentifier>> retMap = new HashMap<>();
+ // Suppress the exception here because it is valid that the entry does not
+ // exist
+ List<String> subclusters = null;
+ try {
+ subclusters = listDirRegistry(this.registry, this.user,
+ getRegistryKey(appId, null), false);
+ } catch (YarnException e) {
+ LOG.warn("Unexpected exception from listDirRegistry", e);
+ }
+
+ if (subclusters == null) {
+ LOG.info("Application {} does not exist in registry", appId);
+ return retMap;
+ }
+
+ // Read the amrmToken for each sub-cluster with an existing UAM
+ for (String scId : subclusters) {
+ LOG.info("Reading amrmToken for subcluster {} for {}", scId, appId);
+ String key = getRegistryKey(appId, scId);
+ try {
+ String tokenString = readRegistry(this.registry, this.user, key, true);
+ if (tokenString == null) {
+ throw new YarnException("Null string from readRegistry key " + key);
+ }
+ Token<AMRMTokenIdentifier> amrmToken = new Token<>();
+ amrmToken.decodeFromUrlString(tokenString);
+ // Clear the service field, as if RM just issued the token
+ amrmToken.setService(new Text());
+
+ retMap.put(scId, amrmToken);
+ } catch (Exception e) {
+ LOG.error("Failed reading registry key " + key
+ + ", skipping subcluster " + scId, e);
+ }
+ }
+
+ // Override existing map if there
+ this.appSubClusterTokenMap.put(appId, new ConcurrentHashMap<>(retMap));
+ return retMap;
+ }
+
+ /**
+ * Remove an application from registry.
+ *
+ * @param appId application id
+ */
+ public void removeAppFromRegistry(ApplicationId appId) {
+ Map<String, Token<AMRMTokenIdentifier>> subClusterTokenMap =
+ this.appSubClusterTokenMap.get(appId);
+ LOG.info("Removing all registry entries for {}", appId);
+
+ if (subClusterTokenMap == null || subClusterTokenMap.size() == 0) {
+ return;
+ }
+
+ // Lastly remove the application directory
+ String key = getRegistryKey(appId, null);
+ try {
+ removeKeyRegistry(this.registry, this.user, key, true, true);
+ subClusterTokenMap.clear();
+ } catch (YarnException e) {
+ LOG.error("Failed removing registry directory key " + key, e);
+ }
+ }
+
+ private String getRegistryKey(ApplicationId appId, String fileName) {
+ if (appId == null) {
+ return this.registryBaseDir;
+ }
+ if (fileName == null) {
+ return this.registryBaseDir + appId.toString();
+ }
+ return this.registryBaseDir + appId.toString() + "/" + fileName;
+ }
+
+ private String readRegistry(final RegistryOperations registryImpl,
+ UserGroupInformation ugi, final String key, final boolean throwIfFails)
+ throws YarnException {
+ // Use the ugi loaded with app credentials to access registry
+ String result = ugi.doAs(new PrivilegedAction<String>() {
+ @Override
+ public String run() {
+ try {
+ ServiceRecord value = registryImpl.resolve(key);
+ if (value != null) {
+ return value.description;
+ }
+ } catch (Throwable e) {
+ if (throwIfFails) {
+ LOG.error("Registry resolve key " + key + " failed", e);
+ }
+ }
+ return null;
+ }
+ });
+ if (result == null && throwIfFails) {
+ throw new YarnException("Registry resolve key " + key + " failed");
+ }
+ return result;
+ }
+
+ private void removeKeyRegistry(final RegistryOperations registryImpl,
+ UserGroupInformation ugi, final String key, final boolean recursive,
+ final boolean throwIfFails) throws YarnException {
+ // Use the ugi loaded with app credentials to access registry
+ boolean success = ugi.doAs(new PrivilegedAction<Boolean>() {
+ @Override
+ public Boolean run() {
+ try {
+ registryImpl.delete(key, recursive);
+ return true;
+ } catch (Throwable e) {
+ if (throwIfFails) {
+ LOG.error("Registry remove key " + key + " failed", e);
+ }
+ }
+ return false;
+ }
+ });
+ if (!success && throwIfFails) {
+ throw new YarnException("Registry remove key " + key + " failed");
+ }
+ }
+
+ /**
+ * Write registry entry, override if exists.
+ */
+ private void writeRegistry(final RegistryOperations registryImpl,
+ UserGroupInformation ugi, final String key, final String value,
+ final boolean throwIfFails) throws YarnException {
+
+ final ServiceRecord recordValue = new ServiceRecord();
+ recordValue.description = value;
+ // Use the ugi loaded with app credentials to access registry
+ boolean success = ugi.doAs(new PrivilegedAction<Boolean>() {
+ @Override
+ public Boolean run() {
+ try {
+ registryImpl.bind(key, recordValue, BindFlags.OVERWRITE);
+ return true;
+ } catch (Throwable e) {
+ if (throwIfFails) {
+ LOG.error("Registry write key " + key + " failed", e);
+ }
+ }
+ return false;
+ }
+ });
+ if (!success && throwIfFails) {
+ throw new YarnException("Registry write key " + key + " failed");
+ }
+ }
+
+ /**
+ * List the sub directories in the given directory.
+ */
+ private List<String> listDirRegistry(final RegistryOperations registryImpl,
+ UserGroupInformation ugi, final String key, final boolean throwIfFails)
+ throws YarnException {
+ List<String> result = ugi.doAs(new PrivilegedAction<List<String>>() {
+ @Override
+ public List<String> run() {
+ try {
+ return registryImpl.list(key);
+ } catch (Throwable e) {
+ if (throwIfFails) {
+ LOG.error("Registry list key " + key + " failed", e);
+ }
+ }
+ return null;
+ }
+ });
+ if (result == null && throwIfFails) {
+ throw new YarnException("Registry list key " + key + " failed");
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
index 08aee77..677c4e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedAMPoolManager.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -44,9 +45,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.utils.AMRMClientUtils;
import org.apache.hadoop.yarn.util.AsyncCallback;
import org.slf4j.Logger;
@@ -67,7 +69,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
// Map from uamId to UAM instances
private Map<String, UnmanagedApplicationManager> unmanagedAppMasterMap;
- private Map<String, ApplicationAttemptId> attemptIdMap;
+ private Map<String, ApplicationId> appIdMap;
private ExecutorService threadpool;
@@ -82,7 +84,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
this.threadpool = Executors.newCachedThreadPool();
}
this.unmanagedAppMasterMap = new ConcurrentHashMap<>();
- this.attemptIdMap = new ConcurrentHashMap<>();
+ this.appIdMap = new ConcurrentHashMap<>();
super.serviceStart();
}
@@ -114,7 +116,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
public KillApplicationResponse call() throws Exception {
try {
LOG.info("Force-killing UAM id " + uamId + " for application "
- + attemptIdMap.get(uamId));
+ + appIdMap.get(uamId));
return unmanagedAppMasterMap.remove(uamId).forceKillApplication();
} catch (Exception e) {
LOG.error("Failed to kill unmanaged application master", e);
@@ -132,7 +134,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
LOG.error("Failed to kill unmanaged application master", e);
}
}
- this.attemptIdMap.clear();
+ this.appIdMap.clear();
super.serviceStop();
}
@@ -145,13 +147,18 @@ public class UnmanagedAMPoolManager extends AbstractService {
* @param queueName queue of the application
* @param submitter submitter name of the UAM
* @param appNameSuffix application name suffix for the UAM
+ * @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+ * recovery.
+ * @see ApplicationSubmissionContext
+ * #setKeepContainersAcrossApplicationAttempts(boolean)
* @return uamId for the UAM
* @throws YarnException if registerApplicationMaster fails
* @throws IOException if registerApplicationMaster fails
*/
public String createAndRegisterNewUAM(
RegisterApplicationMasterRequest registerRequest, Configuration conf,
- String queueName, String submitter, String appNameSuffix)
+ String queueName, String submitter, String appNameSuffix,
+ boolean keepContainersAcrossApplicationAttempts)
throws YarnException, IOException {
ApplicationId appId = null;
ApplicationClientProtocol rmClient;
@@ -173,45 +180,93 @@ public class UnmanagedAMPoolManager extends AbstractService {
rmClient = null;
}
- createAndRegisterNewUAM(appId.toString(), registerRequest, conf, appId,
- queueName, submitter, appNameSuffix);
+ // Launch the UAM in RM
+ launchUAM(appId.toString(), conf, appId, queueName, submitter,
+ appNameSuffix, keepContainersAcrossApplicationAttempts);
+
+ // Register the UAM application
+ registerApplicationMaster(appId.toString(), registerRequest);
+
+ // Returns the appId as uamId
return appId.toString();
}
/**
- * Create a new UAM and register the application, using the provided uamId and
- * appId.
+ * Launch a new UAM, using the provided uamId and appId.
*
- * @param uamId identifier for the UAM
- * @param registerRequest RegisterApplicationMasterRequest
+ * @param uamId uam Id
* @param conf configuration for this UAM
* @param appId application id for the UAM
* @param queueName queue of the application
* @param submitter submitter name of the UAM
* @param appNameSuffix application name suffix for the UAM
- * @return RegisterApplicationMasterResponse
- * @throws YarnException if registerApplicationMaster fails
- * @throws IOException if registerApplicationMaster fails
+ * @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+ * recovery.
+ * @see ApplicationSubmissionContext
+ * #setKeepContainersAcrossApplicationAttempts(boolean)
+ * @return UAM token
+ * @throws YarnException if fails
+ * @throws IOException if fails
*/
- public RegisterApplicationMasterResponse createAndRegisterNewUAM(String uamId,
- RegisterApplicationMasterRequest registerRequest, Configuration conf,
+ public Token<AMRMTokenIdentifier> launchUAM(String uamId, Configuration conf,
+ ApplicationId appId, String queueName, String submitter,
+ String appNameSuffix, boolean keepContainersAcrossApplicationAttempts)
+ throws YarnException, IOException {
+
+ if (this.unmanagedAppMasterMap.containsKey(uamId)) {
+ throw new YarnException("UAM " + uamId + " already exists");
+ }
+ UnmanagedApplicationManager uam = createUAM(conf, appId, queueName,
+ submitter, appNameSuffix, keepContainersAcrossApplicationAttempts);
+ // Put the UAM into map first before initializing it to avoid additional UAM
+ // for the same uamId being created concurrently
+ this.unmanagedAppMasterMap.put(uamId, uam);
+
+ Token<AMRMTokenIdentifier> amrmToken = null;
+ try {
+ LOG.info("Launching UAM id {} for application {}", uamId, appId);
+ amrmToken = uam.launchUAM();
+ } catch (Exception e) {
+ // Add the map earlier and remove here if register failed because we want
+ // to make sure there is only one uam instance per uamId at any given time
+ this.unmanagedAppMasterMap.remove(uamId);
+ throw e;
+ }
+
+ this.appIdMap.put(uamId, uam.getAppId());
+ return amrmToken;
+ }
+
+ /**
+ * Re-attach to an existing UAM, using the provided uamIdentifier.
+ *
+ * @param uamId uam Id
+ * @param conf configuration for this UAM
+ * @param appId application id for the UAM
+ * @param queueName queue of the application
+ * @param submitter submitter name of the UAM
+ * @param appNameSuffix application name suffix for the UAM
+ * @param uamToken UAM token
+ * @throws YarnException if fails
+ * @throws IOException if fails
+ */
+ public void reAttachUAM(String uamId, Configuration conf,
ApplicationId appId, String queueName, String submitter,
- String appNameSuffix) throws YarnException, IOException {
+ String appNameSuffix, Token<AMRMTokenIdentifier> uamToken)
+ throws YarnException, IOException {
if (this.unmanagedAppMasterMap.containsKey(uamId)) {
throw new YarnException("UAM " + uamId + " already exists");
}
UnmanagedApplicationManager uam =
- createUAM(conf, appId, queueName, submitter, appNameSuffix);
+ createUAM(conf, appId, queueName, submitter, appNameSuffix, true);
// Put the UAM into map first before initializing it to avoid additional UAM
// for the same uamId being created concurrently
this.unmanagedAppMasterMap.put(uamId, uam);
- RegisterApplicationMasterResponse response = null;
try {
- LOG.info("Creating and registering UAM id {} for application {}", uamId,
- appId);
- response = uam.createAndRegisterApplicationMaster(registerRequest);
+ LOG.info("Reattaching UAM id {} for application {}", uamId, appId);
+ uam.reAttachUAM(uamToken);
} catch (Exception e) {
// Add the map earlier and remove here if register failed because we want
// to make sure there is only one uam instance per uamId at any given time
@@ -219,8 +274,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
throw e;
}
- this.attemptIdMap.put(uamId, uam.getAttemptId());
- return response;
+ this.appIdMap.put(uamId, uam.getAppId());
}
/**
@@ -231,20 +285,42 @@ public class UnmanagedAMPoolManager extends AbstractService {
* @param queueName queue of the application
* @param submitter submitter name of the application
* @param appNameSuffix application name suffix
+ * @param keepContainersAcrossApplicationAttempts keep container flag for UAM
* @return the UAM instance
*/
@VisibleForTesting
protected UnmanagedApplicationManager createUAM(Configuration conf,
ApplicationId appId, String queueName, String submitter,
- String appNameSuffix) {
+ String appNameSuffix, boolean keepContainersAcrossApplicationAttempts) {
return new UnmanagedApplicationManager(conf, appId, queueName, submitter,
- appNameSuffix);
+ appNameSuffix, keepContainersAcrossApplicationAttempts);
+ }
+
+ /**
+ * Register application master for the UAM.
+ *
+ * @param uamId uam Id
+ * @param registerRequest RegisterApplicationMasterRequest
+ * @return register response
+ * @throws YarnException if register fails
+ * @throws IOException if register fails
+ */
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String uamId, RegisterApplicationMasterRequest registerRequest)
+ throws YarnException, IOException {
+ if (!this.unmanagedAppMasterMap.containsKey(uamId)) {
+ throw new YarnException("UAM " + uamId + " does not exist");
+ }
+ LOG.info("Registering UAM id {} for application {}", uamId,
+ this.appIdMap.get(uamId));
+ return this.unmanagedAppMasterMap.get(uamId)
+ .registerApplicationMaster(registerRequest);
}
/**
* AllocateAsync to an UAM.
*
- * @param uamId identifier for the UAM
+ * @param uamId uam Id
* @param request AllocateRequest
* @param callback callback for response
* @throws YarnException if allocate fails
@@ -262,7 +338,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
/**
* Finish an UAM/application.
*
- * @param uamId identifier for the UAM
+ * @param uamId uam Id
* @param request FinishApplicationMasterRequest
* @return FinishApplicationMasterResponse
* @throws YarnException if finishApplicationMaster call fails
@@ -274,14 +350,15 @@ public class UnmanagedAMPoolManager extends AbstractService {
if (!this.unmanagedAppMasterMap.containsKey(uamId)) {
throw new YarnException("UAM " + uamId + " does not exist");
}
- LOG.info("Finishing application for UAM id {} ", uamId);
+ LOG.info("Finishing UAM id {} for application {}", uamId,
+ this.appIdMap.get(uamId));
FinishApplicationMasterResponse response =
this.unmanagedAppMasterMap.get(uamId).finishApplicationMaster(request);
if (response.getIsUnregistered()) {
// Only remove the UAM when the unregister finished
this.unmanagedAppMasterMap.remove(uamId);
- this.attemptIdMap.remove(uamId);
+ this.appIdMap.remove(uamId);
LOG.info("UAM id {} is unregistered", uamId);
}
return response;
@@ -301,7 +378,7 @@ public class UnmanagedAMPoolManager extends AbstractService {
/**
* Return whether an UAM exists.
*
- * @param uamId identifier for the UAM
+ * @param uamId uam Id
* @return UAM exists or not
*/
public boolean hasUAMId(String uamId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
index 6531a75..3f4a110 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/uam/UnmanagedApplicationManager.java
@@ -50,7 +50,9 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -90,7 +92,6 @@ public class UnmanagedApplicationManager {
private AMRequestHandlerThread handlerThread;
private ApplicationMasterProtocol rmProxy;
private ApplicationId applicationId;
- private ApplicationAttemptId attemptId;
private String submitter;
private String appNameSuffix;
private Configuration conf;
@@ -101,9 +102,31 @@ public class UnmanagedApplicationManager {
private ApplicationClientProtocol rmClient;
private long asyncApiPollIntervalMillis;
private RecordFactory recordFactory;
+ private boolean keepContainersAcrossApplicationAttempts;
+ /*
+ * This flag is used as an indication that this method launchUAM/reAttachUAM
+ * is called (and perhaps blocked in initializeUnmanagedAM below due to RM
+ * connection/failover issue and not finished yet). Set the flag before
+ * calling the blocking call to RM.
+ */
+ private boolean connectionInitiated;
+
+ /**
+ * Constructor.
+ *
+ * @param conf configuration
+ * @param appId application Id to use for this UAM
+ * @param queueName the queue of the UAM
+ * @param submitter user name of the app
+ * @param appNameSuffix the app name suffix to use
+ * @param keepContainersAcrossApplicationAttempts keep container flag for UAM
+ * recovery. See {@link ApplicationSubmissionContext
+ * #setKeepContainersAcrossApplicationAttempts(boolean)}
+ */
public UnmanagedApplicationManager(Configuration conf, ApplicationId appId,
- String queueName, String submitter, String appNameSuffix) {
+ String queueName, String submitter, String appNameSuffix,
+ boolean keepContainersAcrossApplicationAttempts) {
Preconditions.checkNotNull(conf, "Configuration cannot be null");
Preconditions.checkNotNull(appId, "ApplicationId cannot be null");
Preconditions.checkNotNull(submitter, "App submitter cannot be null");
@@ -116,6 +139,7 @@ public class UnmanagedApplicationManager {
this.handlerThread = new AMRequestHandlerThread();
this.requestQueue = new LinkedBlockingQueue<>();
this.rmProxy = null;
+ this.connectionInitiated = false;
this.registerRequest = null;
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
this.asyncApiPollIntervalMillis = conf.getLong(
@@ -123,45 +147,84 @@ public class UnmanagedApplicationManager {
YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS,
YarnConfiguration.
DEFAULT_YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_INTERVAL_MS);
+ this.keepContainersAcrossApplicationAttempts =
+ keepContainersAcrossApplicationAttempts;
+ }
+
+ /**
+ * Launch a new UAM in the resource manager.
+ *
+ * @return identifier uam identifier
+ * @throws YarnException if fails
+ * @throws IOException if fails
+ */
+ public Token<AMRMTokenIdentifier> launchUAM()
+ throws YarnException, IOException {
+ this.connectionInitiated = true;
+
+ // Blocking call to RM
+ Token<AMRMTokenIdentifier> amrmToken =
+ initializeUnmanagedAM(this.applicationId);
+
+ // Creates the UAM connection
+ createUAMProxy(amrmToken);
+ return amrmToken;
+ }
+
+ /**
+ * Re-attach to an existing UAM in the resource manager.
+ *
+ * @param amrmToken the UAM token
+ * @throws IOException if re-attach fails
+ * @throws YarnException if re-attach fails
+ */
+ public void reAttachUAM(Token<AMRMTokenIdentifier> amrmToken)
+ throws IOException, YarnException {
+ this.connectionInitiated = true;
+
+ // Creates the UAM connection
+ createUAMProxy(amrmToken);
+ }
+
+ protected void createUAMProxy(Token<AMRMTokenIdentifier> amrmToken)
+ throws IOException {
+ this.userUgi = UserGroupInformation.createProxyUser(
+ this.applicationId.toString(), UserGroupInformation.getCurrentUser());
+ this.rmProxy = createRMProxy(ApplicationMasterProtocol.class, this.conf,
+ this.userUgi, amrmToken);
}
/**
* Registers this {@link UnmanagedApplicationManager} with the resource
* manager.
*
- * @param request the register request
- * @return the register response
+ * @param request RegisterApplicationMasterRequest
+ * @return register response
* @throws YarnException if register fails
* @throws IOException if register fails
*/
- public RegisterApplicationMasterResponse createAndRegisterApplicationMaster(
+ public RegisterApplicationMasterResponse registerApplicationMaster(
RegisterApplicationMasterRequest request)
throws YarnException, IOException {
- // This need to be done first in this method, because it is used as an
- // indication that this method is called (and perhaps blocked due to RM
- // connection and not finished yet)
+ // Save the register request for re-register later
this.registerRequest = request;
- // attemptId will be available after this call
- UnmanagedAMIdentifier identifier =
- initializeUnmanagedAM(this.applicationId);
-
- try {
- this.userUgi = UserGroupInformation.createProxyUser(
- identifier.getAttemptId().toString(),
- UserGroupInformation.getCurrentUser());
- } catch (IOException e) {
- LOG.error("Exception while trying to get current user", e);
- throw new YarnRuntimeException(e);
- }
-
- this.rmProxy = createRMProxy(ApplicationMasterProtocol.class, this.conf,
- this.userUgi, identifier.getToken());
-
- LOG.info("Registering the Unmanaged application master {}", this.attemptId);
+ // Since we have setKeepContainersAcrossApplicationAttempts = true for UAM.
+ // We do not expect application already registered exception here
+ LOG.info("Registering the Unmanaged application master {}",
+ this.applicationId);
RegisterApplicationMasterResponse response =
this.rmProxy.registerApplicationMaster(this.registerRequest);
+ for (Container container : response.getContainersFromPreviousAttempts()) {
+ LOG.info("RegisterUAM returned existing running container "
+ + container.getId());
+ }
+ for (NMToken nmToken : response.getNMTokensFromPreviousAttempts()) {
+ LOG.info("RegisterUAM returned existing NM token for node "
+ + nmToken.getNodeId());
+ }
+
// Only when register succeed that we start the heartbeat thread
this.handlerThread.setUncaughtExceptionHandler(
new HeartBeatThreadUncaughtExceptionHandler());
@@ -187,11 +250,11 @@ public class UnmanagedApplicationManager {
this.handlerThread.shutdown();
if (this.rmProxy == null) {
- if (this.registerRequest != null) {
- // This is possible if the async registerApplicationMaster is still
+ if (this.connectionInitiated) {
+ // This is possible if the async launchUAM is still
// blocked and retrying. Return a dummy response in this case.
LOG.warn("Unmanaged AM still not successfully launched/registered yet."
- + " Stopping the UAM client thread anyways.");
+ + " Stopping the UAM heartbeat thread anyways.");
return FinishApplicationMasterResponse.newInstance(false);
} else {
throw new YarnException("finishApplicationMaster should not "
@@ -199,7 +262,7 @@ public class UnmanagedApplicationManager {
}
}
return AMRMClientUtils.finishAMWithReRegister(request, this.rmProxy,
- this.registerRequest, this.attemptId);
+ this.registerRequest, this.applicationId);
}
/**
@@ -212,7 +275,7 @@ public class UnmanagedApplicationManager {
public KillApplicationResponse forceKillApplication()
throws IOException, YarnException {
KillApplicationRequest request =
- KillApplicationRequest.newInstance(this.attemptId.getApplicationId());
+ KillApplicationRequest.newInstance(this.applicationId);
this.handlerThread.shutdown();
@@ -240,29 +303,29 @@ public class UnmanagedApplicationManager {
LOG.debug("Interrupted while waiting to put on response queue", ex);
}
// Two possible cases why the UAM is not successfully registered yet:
- // 1. registerApplicationMaster is not called at all. Should throw here.
- // 2. registerApplicationMaster is called but hasn't successfully returned.
+ // 1. launchUAM is not called at all. Should throw here.
+ // 2. launchUAM is called but hasn't successfully returned.
//
// In case 2, we have already save the allocate request above, so if the
// registration succeed later, no request is lost.
if (this.rmProxy == null) {
- if (this.registerRequest != null) {
+ if (this.connectionInitiated) {
LOG.info("Unmanaged AM still not successfully launched/registered yet."
+ " Saving the allocate request and send later.");
} else {
throw new YarnException(
- "AllocateAsync should not be called before createAndRegister");
+ "AllocateAsync should not be called before launchUAM");
}
}
}
/**
- * Returns the application attempt id of the UAM.
+ * Returns the application id of the UAM.
*
- * @return attempt id of the UAM
+ * @return application id of the UAM
*/
- public ApplicationAttemptId getAttemptId() {
- return this.attemptId;
+ public ApplicationId getAppId() {
+ return this.applicationId;
}
/**
@@ -287,15 +350,15 @@ public class UnmanagedApplicationManager {
* Launch and initialize an unmanaged AM. First, it creates a new application
* on the RM and negotiates a new attempt id. Then it waits for the RM
* application attempt state to reach YarnApplicationAttemptState.LAUNCHED
- * after which it returns the AM-RM token and the attemptId.
+ * after which it returns the AM-RM token.
*
* @param appId application id
- * @return the UAM identifier
+ * @return the UAM token
* @throws IOException if initialize fails
* @throws YarnException if initialize fails
*/
- protected UnmanagedAMIdentifier initializeUnmanagedAM(ApplicationId appId)
- throws IOException, YarnException {
+ protected Token<AMRMTokenIdentifier> initializeUnmanagedAM(
+ ApplicationId appId) throws IOException, YarnException {
try {
UserGroupInformation appSubmitter =
UserGroupInformation.createRemoteUser(this.submitter);
@@ -306,13 +369,12 @@ public class UnmanagedApplicationManager {
submitUnmanagedApp(appId);
// Monitor the application attempt to wait for launch state
- ApplicationAttemptReport attemptReport = monitorCurrentAppAttempt(appId,
+ monitorCurrentAppAttempt(appId,
EnumSet.of(YarnApplicationState.ACCEPTED,
YarnApplicationState.RUNNING, YarnApplicationState.KILLED,
YarnApplicationState.FAILED, YarnApplicationState.FINISHED),
YarnApplicationAttemptState.LAUNCHED);
- this.attemptId = attemptReport.getApplicationAttemptId();
- return getUAMIdentifier();
+ return getUAMToken();
} finally {
this.rmClient = null;
}
@@ -343,6 +405,8 @@ public class UnmanagedApplicationManager {
submitRequest.setApplicationSubmissionContext(context);
context.setUnmanagedAM(true);
+ context.setKeepContainersAcrossApplicationAttempts(
+ this.keepContainersAcrossApplicationAttempts);
LOG.info("Submitting unmanaged application {}", appId);
this.rmClient.submitApplication(submitRequest);
@@ -374,8 +438,10 @@ public class UnmanagedApplicationManager {
if (appStates.contains(state)) {
if (state != YarnApplicationState.ACCEPTED) {
throw new YarnRuntimeException(
- "Received non-accepted application state: " + state
- + ". Application " + appId + " not the first attempt?");
+ "Received non-accepted application state: " + state + " for "
+ + appId + ". This is likely because this is not the first "
+ + "app attempt in home sub-cluster, and AMRMProxy HA "
+ + "(yarn.nodemanager.amrmproxy.ha.enable) is not enabled.");
}
appAttemptId =
getApplicationReport(appId).getCurrentApplicationAttemptId();
@@ -415,25 +481,25 @@ public class UnmanagedApplicationManager {
}
/**
- * Gets the identifier of the unmanaged AM.
+ * Gets the amrmToken of the unmanaged AM.
*
- * @return the identifier of the unmanaged AM.
+ * @return the amrmToken of the unmanaged AM.
* @throws IOException if getApplicationReport fails
* @throws YarnException if getApplicationReport fails
*/
- protected UnmanagedAMIdentifier getUAMIdentifier()
+ protected Token<AMRMTokenIdentifier> getUAMToken()
throws IOException, YarnException {
Token<AMRMTokenIdentifier> token = null;
org.apache.hadoop.yarn.api.records.Token amrmToken =
- getApplicationReport(this.attemptId.getApplicationId()).getAMRMToken();
+ getApplicationReport(this.applicationId).getAMRMToken();
if (amrmToken != null) {
token = ConverterUtils.convertFromYarn(amrmToken, (Text) null);
} else {
LOG.warn(
"AMRMToken not found in the application report for application: {}",
- this.attemptId.getApplicationId());
+ this.applicationId);
}
- return new UnmanagedAMIdentifier(this.attemptId, token);
+ return token;
}
private ApplicationReport getApplicationReport(ApplicationId appId)
@@ -445,29 +511,6 @@ public class UnmanagedApplicationManager {
}
/**
- * Data structure that encapsulates the application attempt identifier and the
- * AMRMTokenIdentifier. Make it public because clients with HA need it.
- */
- public static class UnmanagedAMIdentifier {
- private ApplicationAttemptId attemptId;
- private Token<AMRMTokenIdentifier> token;
-
- public UnmanagedAMIdentifier(ApplicationAttemptId attemptId,
- Token<AMRMTokenIdentifier> token) {
- this.attemptId = attemptId;
- this.token = token;
- }
-
- public ApplicationAttemptId getAttemptId() {
- return this.attemptId;
- }
-
- public Token<AMRMTokenIdentifier> getToken() {
- return this.token;
- }
- }
-
- /**
* Data structure that encapsulates AllocateRequest and AsyncCallback
* instance.
*/
@@ -549,8 +592,10 @@ public class UnmanagedApplicationManager {
}
request.setResponseId(lastResponseId);
+
AllocateResponse response = AMRMClientUtils.allocateWithReRegister(
- request, rmProxy, registerRequest, attemptId);
+ request, rmProxy, registerRequest, applicationId);
+
if (response == null) {
throw new YarnException("Null allocateResponse from allocate");
}
@@ -578,18 +623,17 @@ public class UnmanagedApplicationManager {
LOG.debug("Interrupted while waiting for queue", ex);
}
} catch (IOException ex) {
- LOG.warn(
- "IO Error occurred while processing heart beat for " + attemptId,
- ex);
+ LOG.warn("IO Error occurred while processing heart beat for "
+ + applicationId, ex);
} catch (Throwable ex) {
LOG.warn(
- "Error occurred while processing heart beat for " + attemptId,
+ "Error occurred while processing heart beat for " + applicationId,
ex);
}
}
LOG.info("UnmanagedApplicationManager has been stopped for {}. "
- + "AMRequestHandlerThread thread is exiting", attemptId);
+ + "AMRequestHandlerThread thread is exiting", applicationId);
}
}
@@ -600,8 +644,8 @@ public class UnmanagedApplicationManager {
implements UncaughtExceptionHandler {
@Override
public void uncaughtException(Thread t, Throwable e) {
- LOG.error("Heartbeat thread {} for application attempt {} crashed!",
- t.getName(), attemptId, e);
+ LOG.error("Heartbeat thread {} for application {} crashed!",
+ t.getName(), applicationId, e);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/AMRMClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/AMRMClientUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/AMRMClientUtils.java
index 7993bd8..3cecdca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/AMRMClientUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/AMRMClientUtils.java
@@ -36,7 +36,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
@@ -63,16 +63,16 @@ public final class AMRMClientUtils {
/**
* Handle ApplicationNotRegistered exception and re-register.
*
- * @param attemptId app attemptId
+ * @param appId application Id
* @param rmProxy RM proxy instance
* @param registerRequest the AM re-register request
* @throws YarnException if re-register fails
*/
public static void handleNotRegisteredExceptionAndReRegister(
- ApplicationAttemptId attemptId, ApplicationMasterProtocol rmProxy,
+ ApplicationId appId, ApplicationMasterProtocol rmProxy,
RegisterApplicationMasterRequest registerRequest) throws YarnException {
LOG.info("App attempt {} not registered, most likely due to RM failover. "
- + " Trying to re-register.", attemptId);
+ + " Trying to re-register.", appId);
try {
rmProxy.registerApplicationMaster(registerRequest);
} catch (Exception e) {
@@ -93,25 +93,24 @@ public final class AMRMClientUtils {
* @param request allocate request
* @param rmProxy RM proxy
* @param registerRequest the register request for re-register
- * @param attemptId application attempt id
+ * @param appId application id
* @return allocate response
* @throws YarnException if RM call fails
* @throws IOException if RM call fails
*/
public static AllocateResponse allocateWithReRegister(AllocateRequest request,
ApplicationMasterProtocol rmProxy,
- RegisterApplicationMasterRequest registerRequest,
- ApplicationAttemptId attemptId) throws YarnException, IOException {
+ RegisterApplicationMasterRequest registerRequest, ApplicationId appId)
+ throws YarnException, IOException {
try {
return rmProxy.allocate(request);
} catch (ApplicationMasterNotRegisteredException e) {
- handleNotRegisteredExceptionAndReRegister(attemptId, rmProxy,
+ handleNotRegisteredExceptionAndReRegister(appId, rmProxy,
registerRequest);
// reset responseId after re-register
request.setResponseId(0);
// retry allocate
- return allocateWithReRegister(request, rmProxy, registerRequest,
- attemptId);
+ return allocateWithReRegister(request, rmProxy, registerRequest, appId);
}
}
@@ -123,23 +122,22 @@ public final class AMRMClientUtils {
* @param request finishApplicationMaster request
* @param rmProxy RM proxy
* @param registerRequest the register request for re-register
- * @param attemptId application attempt id
+ * @param appId application id
* @return finishApplicationMaster response
* @throws YarnException if RM call fails
* @throws IOException if RM call fails
*/
public static FinishApplicationMasterResponse finishAMWithReRegister(
FinishApplicationMasterRequest request, ApplicationMasterProtocol rmProxy,
- RegisterApplicationMasterRequest registerRequest,
- ApplicationAttemptId attemptId) throws YarnException, IOException {
+ RegisterApplicationMasterRequest registerRequest, ApplicationId appId)
+ throws YarnException, IOException {
try {
return rmProxy.finishApplicationMaster(request);
} catch (ApplicationMasterNotRegisteredException ex) {
- handleNotRegisteredExceptionAndReRegister(attemptId, rmProxy,
+ handleNotRegisteredExceptionAndReRegister(appId, rmProxy,
registerRequest);
// retry finishAM after re-register
- return finishAMWithReRegister(request, rmProxy, registerRequest,
- attemptId);
+ return finishAMWithReRegister(request, rmProxy, registerRequest, appId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 628c781..b5727aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -111,6 +111,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
@@ -177,10 +178,9 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
LoggerFactory.getLogger(MockResourceManagerFacade.class);
private HashSet<ApplicationId> applicationMap = new HashSet<>();
- private HashMap<String, List<ContainerId>> applicationContainerIdMap =
- new HashMap<String, List<ContainerId>>();
- private HashMap<ContainerId, Container> allocatedContainerMap =
- new HashMap<ContainerId, Container>();
+ private HashSet<ApplicationId> keepContainerOnUams = new HashSet<>();
+ private HashMap<ApplicationAttemptId, List<ContainerId>>
+ applicationContainerIdMap = new HashMap<>();
private AtomicInteger containerIndex = new AtomicInteger(0);
private Configuration conf;
private int subClusterId;
@@ -221,7 +221,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
this.isRunning = mode;
}
- private static String getAppIdentifier() throws IOException {
+ private static ApplicationAttemptId getAppIdentifier() throws IOException {
AMRMTokenIdentifier result = null;
UserGroupInformation remoteUgi = UserGroupInformation.getCurrentUser();
Set<TokenIdentifier> tokenIds = remoteUgi.getTokenIdentifiers();
@@ -231,7 +231,8 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
break;
}
}
- return result != null ? result.getApplicationAttemptId().toString() : "";
+ return result != null ? result.getApplicationAttemptId()
+ : ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 0), 0);
}
private void validateRunning() throws ConnectException {
@@ -246,19 +247,32 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
throws YarnException, IOException {
validateRunning();
-
- String amrmToken = getAppIdentifier();
- LOG.info("Registering application attempt: " + amrmToken);
+ ApplicationAttemptId attemptId = getAppIdentifier();
+ LOG.info("Registering application attempt: " + attemptId);
shouldReRegisterNext = false;
+ List<Container> containersFromPreviousAttempt = null;
+
synchronized (applicationContainerIdMap) {
- if (applicationContainerIdMap.containsKey(amrmToken)) {
- throw new InvalidApplicationMasterRequestException(
- AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE);
+ if (applicationContainerIdMap.containsKey(attemptId)) {
+ if (keepContainerOnUams.contains(attemptId.getApplicationId())) {
+ // For UAM with the keepContainersFromPreviousAttempt flag, return all
+ // running containers
+ containersFromPreviousAttempt = new ArrayList<>();
+ for (ContainerId containerId : applicationContainerIdMap
+ .get(attemptId)) {
+ containersFromPreviousAttempt.add(Container.newInstance(containerId,
+ null, null, null, null, null));
+ }
+ } else {
+ throw new InvalidApplicationMasterRequestException(
+ AMRMClientUtils.APP_ALREADY_REGISTERED_MESSAGE);
+ }
+ } else {
+ // Keep track of the containers that are returned to this application
+ applicationContainerIdMap.put(attemptId, new ArrayList<ContainerId>());
}
- // Keep track of the containers that are returned to this application
- applicationContainerIdMap.put(amrmToken, new ArrayList<ContainerId>());
}
// Make sure we wait for certain test cases last in the method
@@ -278,7 +292,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
}
return RegisterApplicationMasterResponse.newInstance(null, null, null, null,
- null, request.getHost(), null);
+ containersFromPreviousAttempt, request.getHost(), null);
}
@Override
@@ -288,8 +302,8 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
validateRunning();
- String amrmToken = getAppIdentifier();
- LOG.info("Finishing application attempt: " + amrmToken);
+ ApplicationAttemptId attemptId = getAppIdentifier();
+ LOG.info("Finishing application attempt: " + attemptId);
if (shouldReRegisterNext) {
String message = "AM is not registered, should re-register.";
@@ -299,12 +313,9 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
synchronized (applicationContainerIdMap) {
// Remove the containers that were being tracked for this application
- Assert.assertTrue("The application id is NOT registered: " + amrmToken,
- applicationContainerIdMap.containsKey(amrmToken));
- List<ContainerId> ids = applicationContainerIdMap.remove(amrmToken);
- for (ContainerId c : ids) {
- allocatedContainerMap.remove(c);
- }
+ Assert.assertTrue("The application id is NOT registered: " + attemptId,
+ applicationContainerIdMap.containsKey(attemptId));
+ applicationContainerIdMap.remove(attemptId);
}
return FinishApplicationMasterResponse.newInstance(
@@ -334,8 +345,8 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
+ "askList and releaseList in the same heartbeat");
}
- String amrmToken = getAppIdentifier();
- LOG.info("Allocate from application attempt: " + amrmToken);
+ ApplicationAttemptId attemptId = getAppIdentifier();
+ LOG.info("Allocate from application attempt: " + attemptId);
if (shouldReRegisterNext) {
String message = "AM is not registered, should re-register.";
@@ -367,16 +378,16 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
// will need it in future
Assert.assertTrue(
"The application id is Not registered before allocate(): "
- + amrmToken,
- applicationContainerIdMap.containsKey(amrmToken));
- List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
+ + attemptId,
+ applicationContainerIdMap.containsKey(attemptId));
+ List<ContainerId> ids = applicationContainerIdMap.get(attemptId);
ids.add(containerId);
- this.allocatedContainerMap.put(containerId, container);
}
}
}
}
+ List<ContainerStatus> completedList = new ArrayList<>();
if (request.getReleaseList() != null
&& request.getReleaseList().size() > 0) {
LOG.info("Releasing containers: " + request.getReleaseList().size());
@@ -384,9 +395,9 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
Assert
.assertTrue(
"The application id is not registered before allocate(): "
- + amrmToken,
- applicationContainerIdMap.containsKey(amrmToken));
- List<ContainerId> ids = applicationContainerIdMap.get(amrmToken);
+ + attemptId,
+ applicationContainerIdMap.containsKey(attemptId));
+ List<ContainerId> ids = applicationContainerIdMap.get(attemptId);
for (ContainerId id : request.getReleaseList()) {
boolean found = false;
@@ -402,18 +413,8 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
+ conf.get("AMRMTOKEN"), found);
ids.remove(id);
-
- // Return the released container back to the AM with new fake Ids. The
- // test case does not care about the IDs. The IDs are faked because
- // otherwise the LRM will throw duplication identifier exception. This
- // returning of fake containers is ONLY done for testing purpose - for
- // the test code to get confirmation that the sub-cluster resource
- // managers received the release request
- ContainerId fakeContainerId = ContainerId.newInstance(
- getApplicationAttemptId(1), containerIndex.incrementAndGet());
- Container fakeContainer = allocatedContainerMap.get(id);
- fakeContainer.setId(fakeContainerId);
- containerList.add(fakeContainer);
+ completedList.add(
+ ContainerStatus.newInstance(id, ContainerState.COMPLETE, "", 0));
}
}
}
@@ -424,9 +425,9 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
// Always issue a new AMRMToken as if RM rolled master key
Token newAMRMToken = Token.newInstance(new byte[0], "", new byte[0], "");
- return AllocateResponse.newInstance(0, new ArrayList<ContainerStatus>(),
- containerList, new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC,
- 1, null, new ArrayList<NMToken>(), newAMRMToken,
+ return AllocateResponse.newInstance(0, completedList, containerList,
+ new ArrayList<NodeReport>(), null, AMCommand.AM_RESYNC, 1, null,
+ new ArrayList<NMToken>(), newAMRMToken,
new ArrayList<UpdatedContainer>());
}
@@ -443,6 +444,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
report.setApplicationId(request.getApplicationId());
report.setCurrentApplicationAttemptId(
ApplicationAttemptId.newInstance(request.getApplicationId(), 1));
+ report.setAMRMToken(Token.newInstance(new byte[0], "", new byte[0], ""));
response.setApplicationReport(report);
return response;
}
@@ -486,6 +488,12 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
}
LOG.info("Application submitted: " + appId);
applicationMap.add(appId);
+
+ if (request.getApplicationSubmissionContext().getUnmanagedAM()
+ || request.getApplicationSubmissionContext()
+ .getKeepContainersAcrossApplicationAttempts()) {
+ keepContainerOnUams.add(appId);
+ }
return SubmitApplicationResponse.newInstance();
}
@@ -502,6 +510,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
throw new ApplicationNotFoundException(
"Trying to kill an absent application: " + appId);
}
+ keepContainerOnUams.remove(appId);
}
LOG.info("Force killing application: " + appId);
return KillApplicationResponse.newInstance(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java
new file mode 100644
index 0000000..42be851
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationRegistryClient.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.utils;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.registry.client.impl.FSRegistryOperationsService;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Unit test for FederationRegistryClient.
+ */
+public class TestFederationRegistryClient {
+ private Configuration conf;
+ private UserGroupInformation user;
+ private RegistryOperations registry;
+ private FederationRegistryClient registryClient;
+
+ @Before
+ public void setup() throws Exception {
+ this.conf = new YarnConfiguration();
+
+ this.registry = new FSRegistryOperationsService();
+ this.registry.init(this.conf);
+ this.registry.start();
+
+ this.user = UserGroupInformation.getCurrentUser();
+ this.registryClient =
+ new FederationRegistryClient(this.conf, this.registry, this.user);
+ this.registryClient.cleanAllApplications();
+ Assert.assertEquals(0, this.registryClient.getAllApplications().size());
+ }
+
+ @After
+ public void breakDown() {
+ registryClient.cleanAllApplications();
+ Assert.assertEquals(0, registryClient.getAllApplications().size());
+ registry.stop();
+ }
+
+ @Test
+ public void testBasicCase() {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ String scId1 = "subcluster1";
+ String scId2 = "subcluster2";
+
+ this.registryClient.writeAMRMTokenForUAM(appId, scId1,
+ new Token<AMRMTokenIdentifier>());
+ this.registryClient.writeAMRMTokenForUAM(appId, scId2,
+ new Token<AMRMTokenIdentifier>());
+ // Duplicate entry, should overwrite
+ this.registryClient.writeAMRMTokenForUAM(appId, scId1,
+ new Token<AMRMTokenIdentifier>());
+
+ Assert.assertEquals(1, this.registryClient.getAllApplications().size());
+ Assert.assertEquals(2,
+ this.registryClient.loadStateFromRegistry(appId).size());
+
+ this.registryClient.removeAppFromRegistry(appId);
+
+ Assert.assertEquals(0, this.registryClient.getAllApplications().size());
+ Assert.assertEquals(0,
+ this.registryClient.loadStateFromRegistry(appId).size());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
index 9159cf7..5848d3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/uam/TestUnmanagedApplicationManager.java
@@ -65,7 +65,7 @@ public class TestUnmanagedApplicationManager {
ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1);
uam = new TestableUnmanagedApplicationManager(conf,
- attemptId.getApplicationId(), null, "submitter", "appNameSuffix");
+ attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true);
}
protected void waitForCallBackCountAndCheckZeroPending(
@@ -88,7 +88,8 @@ public class TestUnmanagedApplicationManager {
public void testBasicUsage()
throws YarnException, IOException, InterruptedException {
- createAndRegisterApplicationMaster(
+ launchUAM(attemptId);
+ registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
allocateAsync(AllocateRequest.newInstance(0, 0, null, null, null), callback,
@@ -102,11 +103,48 @@ public class TestUnmanagedApplicationManager {
attemptId);
}
+ /*
+ * Test re-attaching of an existing UAM. This is for HA of UAM client.
+ */
+ @Test(timeout = 5000)
+ public void testUAMReAttach()
+ throws YarnException, IOException, InterruptedException {
+
+ launchUAM(attemptId);
+ registerApplicationMaster(
+ RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
+
+ allocateAsync(AllocateRequest.newInstance(0, 0, null, null, null), callback,
+ attemptId);
+ // Wait for outstanding async allocate callback
+ waitForCallBackCountAndCheckZeroPending(callback, 1);
+
+ MockResourceManagerFacade rmProxy = uam.getRMProxy();
+ uam = new TestableUnmanagedApplicationManager(conf,
+ attemptId.getApplicationId(), null, "submitter", "appNameSuffix", true);
+ uam.setRMProxy(rmProxy);
+
+ reAttachUAM(null, attemptId);
+ registerApplicationMaster(
+ RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
+
+ allocateAsync(AllocateRequest.newInstance(0, 0, null, null, null), callback,
+ attemptId);
+
+ // Wait for outstanding async allocate callback
+ waitForCallBackCountAndCheckZeroPending(callback, 2);
+
+ finishApplicationMaster(
+ FinishApplicationMasterRequest.newInstance(null, null, null),
+ attemptId);
+ }
+
@Test(timeout = 5000)
public void testReRegister()
throws YarnException, IOException, InterruptedException {
- createAndRegisterApplicationMaster(
+ launchUAM(attemptId);
+ registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
uam.setShouldReRegisterNext();
@@ -137,7 +175,8 @@ public class TestUnmanagedApplicationManager {
@Override
public void run() {
try {
- createAndRegisterApplicationMaster(
+ launchUAM(attemptId);
+ registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance(null, 1001, null),
attemptId);
} catch (Exception e) {
@@ -221,7 +260,8 @@ public class TestUnmanagedApplicationManager {
@Test
public void testForceKill()
throws YarnException, IOException, InterruptedException {
- createAndRegisterApplicationMaster(
+ launchUAM(attemptId);
+ registerApplicationMaster(
RegisterApplicationMasterRequest.newInstance(null, 0, null), attemptId);
uam.forceKillApplication();
@@ -241,19 +281,40 @@ public class TestUnmanagedApplicationManager {
return ugi;
}
- protected RegisterApplicationMasterResponse
- createAndRegisterApplicationMaster(
- final RegisterApplicationMasterRequest request,
- ApplicationAttemptId appAttemptId)
- throws YarnException, IOException, InterruptedException {
+ protected Token<AMRMTokenIdentifier> launchUAM(
+ ApplicationAttemptId appAttemptId)
+ throws IOException, InterruptedException {
+ return getUGIWithToken(appAttemptId)
+ .doAs(new PrivilegedExceptionAction<Token<AMRMTokenIdentifier>>() {
+ @Override
+ public Token<AMRMTokenIdentifier> run() throws Exception {
+ return uam.launchUAM();
+ }
+ });
+ }
+
+ protected void reAttachUAM(final Token<AMRMTokenIdentifier> uamToken,
+ ApplicationAttemptId appAttemptId)
+ throws IOException, InterruptedException {
+ getUGIWithToken(appAttemptId).doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Token<AMRMTokenIdentifier> run() throws Exception {
+ uam.reAttachUAM(uamToken);
+ return null;
+ }
+ });
+ }
+
+ protected RegisterApplicationMasterResponse registerApplicationMaster(
+ final RegisterApplicationMasterRequest request,
+ ApplicationAttemptId appAttemptId)
+ throws YarnException, IOException, InterruptedException {
return getUGIWithToken(appAttemptId).doAs(
new PrivilegedExceptionAction<RegisterApplicationMasterResponse>() {
@Override
public RegisterApplicationMasterResponse run()
throws YarnException, IOException {
- RegisterApplicationMasterResponse response =
- uam.createAndRegisterApplicationMaster(request);
- return response;
+ return uam.registerApplicationMaster(request);
}
});
}
@@ -311,8 +372,9 @@ public class TestUnmanagedApplicationManager {
public TestableUnmanagedApplicationManager(Configuration conf,
ApplicationId appId, String queueName, String submitter,
- String appNameSuffix) {
- super(conf, appId, queueName, submitter, appNameSuffix);
+ String appNameSuffix, boolean keepContainersAcrossApplicationAttempts) {
+ super(conf, appId, queueName, submitter, appNameSuffix,
+ keepContainersAcrossApplicationAttempts);
}
@SuppressWarnings("unchecked")
@@ -330,6 +392,14 @@ public class TestUnmanagedApplicationManager {
rmProxy.setShouldReRegisterNext();
}
}
+
+ public MockResourceManagerFacade getRMProxy() {
+ return rmProxy;
+ }
+
+ public void setRMProxy(MockResourceManagerFacade proxy) {
+ this.rmProxy = proxy;
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
index c355a8b..92afcb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContext.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.nodemanager.amrmproxy;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -67,4 +69,18 @@ public interface AMRMProxyApplicationContext {
*/
Context getNMCotext();
+ /**
+ * Gets the credentials of this application.
+ *
+ * @return the credentials.
+ */
+ Credentials getCredentials();
+
+ /**
+ * Gets the registry client.
+ *
+ * @return the registry.
+ */
+ RegistryOperations getRegistryClient();
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5f66888/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
index 9938b37..8a02095 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyApplicationContextImpl.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -42,6 +44,8 @@ public class AMRMProxyApplicationContextImpl implements
private Integer localTokenKeyId;
private Token<AMRMTokenIdentifier> amrmToken;
private Token<AMRMTokenIdentifier> localToken;
+ private Credentials credentials;
+ private RegistryOperations registry;
/**
* Create an instance of the AMRMProxyApplicationContext.
@@ -52,17 +56,23 @@ public class AMRMProxyApplicationContextImpl implements
* @param user user name of the application
* @param amrmToken amrmToken issued by RM
* @param localToken amrmToken issued by AMRMProxy
+ * @param credentials application credentials
+ * @param registry Yarn Registry client
*/
- public AMRMProxyApplicationContextImpl(Context nmContext,
- Configuration conf, ApplicationAttemptId applicationAttemptId,
- String user, Token<AMRMTokenIdentifier> amrmToken,
- Token<AMRMTokenIdentifier> localToken) {
+ @SuppressWarnings("checkstyle:parameternumber")
+ public AMRMProxyApplicationContextImpl(Context nmContext, Configuration conf,
+ ApplicationAttemptId applicationAttemptId, String user,
+ Token<AMRMTokenIdentifier> amrmToken,
+ Token<AMRMTokenIdentifier> localToken, Credentials credentials,
+ RegistryOperations registry) {
this.nmContext = nmContext;
this.conf = conf;
this.applicationAttemptId = applicationAttemptId;
this.user = user;
this.amrmToken = amrmToken;
this.localToken = localToken;
+ this.credentials = credentials;
+ this.registry = registry;
}
@Override
@@ -88,11 +98,14 @@ public class AMRMProxyApplicationContextImpl implements
/**
* Sets the application's AMRMToken.
*
- * @param amrmToken amrmToken issued by RM
+ * @param amrmToken the new amrmToken from RM
+ * @return whether the saved token is updated to a different value
*/
- public synchronized void setAMRMToken(
+ public synchronized boolean setAMRMToken(
Token<AMRMTokenIdentifier> amrmToken) {
+ Token<AMRMTokenIdentifier> oldValue = this.amrmToken;
this.amrmToken = amrmToken;
+ return !this.amrmToken.equals(oldValue);
}
@Override
@@ -134,4 +147,14 @@ public class AMRMProxyApplicationContextImpl implements
public Context getNMCotext() {
return nmContext;
}
+
+ @Override
+ public Credentials getCredentials() {
+ return this.credentials;
+ }
+
+ @Override
+ public RegistryOperations getRegistryClient() {
+ return this.registry;
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/49] hadoop git commit: HDFS-12847. Regenerate editsStored and
editsStored.xml in HDFS tests. Contributed by Lei (Eddy) Xu.
Posted by su...@apache.org.
HDFS-12847. Regenerate editsStored and editsStored.xml in HDFS tests. Contributed by Lei (Eddy) Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/785732c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/785732c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/785732c1
Branch: refs/heads/YARN-5881
Commit: 785732c13e2ebe9f27350b6be82eb2fb782d7dc4
Parents: d42a336
Author: Lei Xu <le...@apache.org>
Authored: Wed Nov 22 10:19:58 2017 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Wed Nov 22 10:22:32 2017 -0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/test/resources/editsStored | Bin 6293 -> 6753 bytes
.../src/test/resources/editsStored.xml | 750 +++++++++++--------
2 files changed, 423 insertions(+), 327 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/785732c1/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index 8029575..3f2817a 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
http://git-wip-us.apache.org/repos/asf/hadoop/blob/785732c1/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 0a1c25e..2a57c73 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
- <EXPIRY_DATE>1423097579620</EXPIRY_DATE>
- <KEY>ef3f2032e2797e8e</KEY>
+ <EXPIRY_DATE>1512000829976</EXPIRY_DATE>
+ <KEY>e7457bcc6ab95a84</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
- <EXPIRY_DATE>1423097579622</EXPIRY_DATE>
- <KEY>b978ed731a0b4a65</KEY>
+ <EXPIRY_DATE>1512000829980</EXPIRY_DATE>
+ <KEY>07cc38caf6c47bb4</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -37,19 +37,19 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380345</MTIME>
- <ATIME>1422406380345</ATIME>
+ <MTIME>1511309632199</MTIME>
+ <ATIME>1511309632199</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>6</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>5</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -60,14 +60,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380369</MTIME>
- <ATIME>1422406380345</ATIME>
+ <MTIME>1511309632248</MTIME>
+ <ATIME>1511309632199</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -78,11 +78,11 @@
<DATA>
<TXID>6</TXID>
<PATH>/file_create</PATH>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<NEWBLOCK>false</NEWBLOCK>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>8</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>7</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
@@ -93,23 +93,118 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380376</MTIME>
- <ATIME>1422406380345</ATIME>
+ <MTIME>1511309632263</MTIME>
+ <ATIME>1511309632199</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
</DATA>
</RECORD>
<RECORD>
- <OPCODE>OP_SET_STORAGE_POLICY</OPCODE>
+ <OPCODE>OP_ADD</OPCODE>
<DATA>
<TXID>8</TXID>
+ <LENGTH>0</LENGTH>
+ <INODEID>16387</INODEID>
+ <PATH>/update_blocks</PATH>
+ <REPLICATION>1</REPLICATION>
+ <MTIME>1511309632266</MTIME>
+ <ATIME>1511309632266</ATIME>
+ <BLOCKSIZE>4096</BLOCKSIZE>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
+ <CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
+ <OVERWRITE>true</OVERWRITE>
+ <PERMISSION_STATUS>
+ <USERNAME>lei</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <MODE>420</MODE>
+ </PERMISSION_STATUS>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>9</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
+ <DATA>
+ <TXID>9</TXID>
+ <BLOCK_ID>1073741825</BLOCK_ID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
+ <DATA>
+ <TXID>10</TXID>
+ <GENSTAMPV2>1001</GENSTAMPV2>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_ADD_BLOCK</OPCODE>
+ <DATA>
+ <TXID>11</TXID>
+ <PATH>/update_blocks</PATH>
+ <BLOCK>
+ <BLOCK_ID>1073741825</BLOCK_ID>
+ <NUM_BYTES>0</NUM_BYTES>
+ <GENSTAMP>1001</GENSTAMP>
+ </BLOCK>
+ <RPC_CLIENTID/>
+ <RPC_CALLID>-2</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_UPDATE_BLOCKS</OPCODE>
+ <DATA>
+ <TXID>12</TXID>
+ <PATH>/update_blocks</PATH>
+ <BLOCK>
+ <BLOCK_ID>1073741825</BLOCK_ID>
+ <NUM_BYTES>1</NUM_BYTES>
+ <GENSTAMP>1001</GENSTAMP>
+ </BLOCK>
+ <RPC_CLIENTID/>
+ <RPC_CALLID>-2</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_UPDATE_BLOCKS</OPCODE>
+ <DATA>
+ <TXID>13</TXID>
+ <PATH>/update_blocks</PATH>
+ <RPC_CLIENTID/>
+ <RPC_CALLID>-2</RPC_CALLID>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_CLOSE</OPCODE>
+ <DATA>
+ <TXID>14</TXID>
+ <LENGTH>0</LENGTH>
+ <INODEID>0</INODEID>
+ <PATH>/update_blocks</PATH>
+ <REPLICATION>1</REPLICATION>
+ <MTIME>1511309632454</MTIME>
+ <ATIME>1511309632266</ATIME>
+ <BLOCKSIZE>4096</BLOCKSIZE>
+ <CLIENT_NAME/>
+ <CLIENT_MACHINE/>
+ <OVERWRITE>false</OVERWRITE>
+ <PERMISSION_STATUS>
+ <USERNAME>lei</USERNAME>
+ <GROUPNAME>supergroup</GROUPNAME>
+ <MODE>420</MODE>
+ </PERMISSION_STATUS>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_SET_STORAGE_POLICY</OPCODE>
+ <DATA>
+ <TXID>15</TXID>
<PATH>/file_create</PATH>
<POLICYID>7</POLICYID>
</DATA>
@@ -117,36 +212,36 @@
<RECORD>
<OPCODE>OP_RENAME_OLD</OPCODE>
<DATA>
- <TXID>9</TXID>
+ <TXID>16</TXID>
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1422406380383</TIMESTAMP>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>11</RPC_CALLID>
+ <TIMESTAMP>1511309632467</TIMESTAMP>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>17</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_DELETE</OPCODE>
<DATA>
- <TXID>10</TXID>
+ <TXID>17</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
- <TIMESTAMP>1422406380392</TIMESTAMP>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>12</RPC_CALLID>
+ <TIMESTAMP>1511309632480</TIMESTAMP>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>19</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MKDIR</OPCODE>
<DATA>
- <TXID>11</TXID>
+ <TXID>18</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16387</INODEID>
+ <INODEID>16388</INODEID>
<PATH>/directory_mkdir</PATH>
- <TIMESTAMP>1422406380399</TIMESTAMP>
+ <TIMESTAMP>1511309632488</TIMESTAMP>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@@ -155,94 +250,94 @@
<RECORD>
<OPCODE>OP_ALLOW_SNAPSHOT</OPCODE>
<DATA>
- <TXID>12</TXID>
+ <TXID>19</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_DISALLOW_SNAPSHOT</OPCODE>
<DATA>
- <TXID>13</TXID>
+ <TXID>20</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOW_SNAPSHOT</OPCODE>
<DATA>
- <TXID>14</TXID>
+ <TXID>21</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_CREATE_SNAPSHOT</OPCODE>
<DATA>
- <TXID>15</TXID>
+ <TXID>22</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>17</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>24</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_RENAME_SNAPSHOT</OPCODE>
<DATA>
- <TXID>16</TXID>
+ <TXID>23</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>18</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>25</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_DELETE_SNAPSHOT</OPCODE>
<DATA>
- <TXID>17</TXID>
+ <TXID>24</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>19</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>26</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>18</TXID>
+ <TXID>25</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16388</INODEID>
+ <INODEID>16389</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380423</MTIME>
- <ATIME>1422406380423</ATIME>
+ <MTIME>1511309632528</MTIME>
+ <ATIME>1511309632528</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>20</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>27</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>19</TXID>
+ <TXID>26</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380425</MTIME>
- <ATIME>1422406380423</ATIME>
+ <MTIME>1511309632530</MTIME>
+ <ATIME>1511309632528</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -251,7 +346,7 @@
<RECORD>
<OPCODE>OP_SET_REPLICATION</OPCODE>
<DATA>
- <TXID>20</TXID>
+ <TXID>27</TXID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
</DATA>
@@ -259,7 +354,7 @@
<RECORD>
<OPCODE>OP_SET_PERMISSIONS</OPCODE>
<DATA>
- <TXID>21</TXID>
+ <TXID>28</TXID>
<SRC>/file_create</SRC>
<MODE>511</MODE>
</DATA>
@@ -267,7 +362,7 @@
<RECORD>
<OPCODE>OP_SET_OWNER</OPCODE>
<DATA>
- <TXID>22</TXID>
+ <TXID>29</TXID>
<SRC>/file_create</SRC>
<USERNAME>newOwner</USERNAME>
</DATA>
@@ -275,7 +370,7 @@
<RECORD>
<OPCODE>OP_TIMES</OPCODE>
<DATA>
- <TXID>23</TXID>
+ <TXID>30</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_create</PATH>
<MTIME>1285195527000</MTIME>
@@ -285,7 +380,7 @@
<RECORD>
<OPCODE>OP_SET_QUOTA</OPCODE>
<DATA>
- <TXID>24</TXID>
+ <TXID>31</TXID>
<SRC>/directory_mkdir</SRC>
<NSQUOTA>1000</NSQUOTA>
<DSQUOTA>-1</DSQUOTA>
@@ -294,7 +389,7 @@
<RECORD>
<OPCODE>OP_SET_QUOTA_BY_STORAGETYPE</OPCODE>
<DATA>
- <TXID>25</TXID>
+ <TXID>32</TXID>
<SRC>/directory_mkdir</SRC>
<STORAGETYPE>1</STORAGETYPE>
<DSQUOTA>888</DSQUOTA>
@@ -303,62 +398,62 @@
<RECORD>
<OPCODE>OP_RENAME</OPCODE>
<DATA>
- <TXID>26</TXID>
+ <TXID>33</TXID>
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1422406380442</TIMESTAMP>
- <OPTIONS>NONE</OPTIONS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>28</RPC_CALLID>
+ <TIMESTAMP>1511309632561</TIMESTAMP>
+ <OPTIONS>TO_TRASH</OPTIONS>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>35</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>27</TXID>
+ <TXID>34</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16389</INODEID>
+ <INODEID>16390</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380446</MTIME>
- <ATIME>1422406380446</ATIME>
+ <MTIME>1511309632566</MTIME>
+ <ATIME>1511309632566</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>30</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>37</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>28</TXID>
- <BLOCK_ID>1073741825</BLOCK_ID>
+ <TXID>35</TXID>
+ <BLOCK_ID>1073741826</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>29</TXID>
- <GENSTAMPV2>1001</GENSTAMPV2>
+ <TXID>36</TXID>
+ <GENSTAMPV2>1002</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>30</TXID>
+ <TXID>37</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
- <BLOCK_ID>1073741825</BLOCK_ID>
+ <BLOCK_ID>1073741826</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1001</GENSTAMP>
+ <GENSTAMP>1002</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -367,31 +462,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>31</TXID>
- <BLOCK_ID>1073741826</BLOCK_ID>
+ <TXID>38</TXID>
+ <BLOCK_ID>1073741827</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>32</TXID>
- <GENSTAMPV2>1002</GENSTAMPV2>
+ <TXID>39</TXID>
+ <GENSTAMPV2>1003</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>33</TXID>
+ <TXID>40</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
- <BLOCK_ID>1073741825</BLOCK_ID>
+ <BLOCK_ID>1073741826</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1001</GENSTAMP>
+ <GENSTAMP>1002</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741826</BLOCK_ID>
+ <BLOCK_ID>1073741827</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1002</GENSTAMP>
+ <GENSTAMP>1003</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -400,31 +495,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>34</TXID>
- <BLOCK_ID>1073741827</BLOCK_ID>
+ <TXID>41</TXID>
+ <BLOCK_ID>1073741828</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>35</TXID>
- <GENSTAMPV2>1003</GENSTAMPV2>
+ <TXID>42</TXID>
+ <GENSTAMPV2>1004</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>36</TXID>
+ <TXID>43</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
- <BLOCK_ID>1073741826</BLOCK_ID>
+ <BLOCK_ID>1073741827</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1002</GENSTAMP>
+ <GENSTAMP>1003</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741827</BLOCK_ID>
+ <BLOCK_ID>1073741828</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1003</GENSTAMP>
+ <GENSTAMP>1004</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -433,23 +528,18 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>37</TXID>
+ <TXID>44</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380534</MTIME>
- <ATIME>1422406380446</ATIME>
+ <MTIME>1511309632593</MTIME>
+ <ATIME>1511309632566</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<BLOCK>
- <BLOCK_ID>1073741825</BLOCK_ID>
- <NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1001</GENSTAMP>
- </BLOCK>
- <BLOCK>
<BLOCK_ID>1073741826</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
<GENSTAMP>1002</GENSTAMP>
@@ -459,8 +549,13 @@
<NUM_BYTES>512</NUM_BYTES>
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
+ <BLOCK>
+ <BLOCK_ID>1073741828</BLOCK_ID>
+ <NUM_BYTES>512</NUM_BYTES>
+ <GENSTAMP>1004</GENSTAMP>
+ </BLOCK>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -469,49 +564,49 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>38</TXID>
+ <TXID>45</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16390</INODEID>
+ <INODEID>16391</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380537</MTIME>
- <ATIME>1422406380537</ATIME>
+ <MTIME>1511309632596</MTIME>
+ <ATIME>1511309632596</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>40</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>46</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>39</TXID>
- <BLOCK_ID>1073741828</BLOCK_ID>
+ <TXID>46</TXID>
+ <BLOCK_ID>1073741829</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>40</TXID>
- <GENSTAMPV2>1004</GENSTAMPV2>
+ <TXID>47</TXID>
+ <GENSTAMPV2>1005</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>41</TXID>
+ <TXID>48</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
- <BLOCK_ID>1073741828</BLOCK_ID>
+ <BLOCK_ID>1073741829</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1004</GENSTAMP>
+ <GENSTAMP>1005</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -520,31 +615,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>42</TXID>
- <BLOCK_ID>1073741829</BLOCK_ID>
+ <TXID>49</TXID>
+ <BLOCK_ID>1073741830</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>43</TXID>
- <GENSTAMPV2>1005</GENSTAMPV2>
+ <TXID>50</TXID>
+ <GENSTAMPV2>1006</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>44</TXID>
+ <TXID>51</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
- <BLOCK_ID>1073741828</BLOCK_ID>
+ <BLOCK_ID>1073741829</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1004</GENSTAMP>
+ <GENSTAMP>1005</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741829</BLOCK_ID>
+ <BLOCK_ID>1073741830</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1005</GENSTAMP>
+ <GENSTAMP>1006</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -553,31 +648,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>45</TXID>
- <BLOCK_ID>1073741830</BLOCK_ID>
+ <TXID>52</TXID>
+ <BLOCK_ID>1073741831</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>46</TXID>
- <GENSTAMPV2>1006</GENSTAMPV2>
+ <TXID>53</TXID>
+ <GENSTAMPV2>1007</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>47</TXID>
+ <TXID>54</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
- <BLOCK_ID>1073741829</BLOCK_ID>
+ <BLOCK_ID>1073741830</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1005</GENSTAMP>
+ <GENSTAMP>1006</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741830</BLOCK_ID>
+ <BLOCK_ID>1073741831</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1006</GENSTAMP>
+ <GENSTAMP>1007</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -586,23 +681,18 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>48</TXID>
+ <TXID>55</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380558</MTIME>
- <ATIME>1422406380537</ATIME>
+ <MTIME>1511309632618</MTIME>
+ <ATIME>1511309632596</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<BLOCK>
- <BLOCK_ID>1073741828</BLOCK_ID>
- <NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1004</GENSTAMP>
- </BLOCK>
- <BLOCK>
<BLOCK_ID>1073741829</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
<GENSTAMP>1005</GENSTAMP>
@@ -612,8 +702,13 @@
<NUM_BYTES>512</NUM_BYTES>
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
+ <BLOCK>
+ <BLOCK_ID>1073741831</BLOCK_ID>
+ <NUM_BYTES>512</NUM_BYTES>
+ <GENSTAMP>1007</GENSTAMP>
+ </BLOCK>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -622,49 +717,49 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>49</TXID>
+ <TXID>56</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16391</INODEID>
+ <INODEID>16392</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380560</MTIME>
- <ATIME>1422406380560</ATIME>
+ <MTIME>1511309632620</MTIME>
+ <ATIME>1511309632620</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>49</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>55</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>50</TXID>
- <BLOCK_ID>1073741831</BLOCK_ID>
+ <TXID>57</TXID>
+ <BLOCK_ID>1073741832</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>51</TXID>
- <GENSTAMPV2>1007</GENSTAMPV2>
+ <TXID>58</TXID>
+ <GENSTAMPV2>1008</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>52</TXID>
+ <TXID>59</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
- <BLOCK_ID>1073741831</BLOCK_ID>
+ <BLOCK_ID>1073741832</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1007</GENSTAMP>
+ <GENSTAMP>1008</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -673,31 +768,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>53</TXID>
- <BLOCK_ID>1073741832</BLOCK_ID>
+ <TXID>60</TXID>
+ <BLOCK_ID>1073741833</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>54</TXID>
- <GENSTAMPV2>1008</GENSTAMPV2>
+ <TXID>61</TXID>
+ <GENSTAMPV2>1009</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>55</TXID>
+ <TXID>62</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
- <BLOCK_ID>1073741831</BLOCK_ID>
+ <BLOCK_ID>1073741832</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1007</GENSTAMP>
+ <GENSTAMP>1008</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741832</BLOCK_ID>
+ <BLOCK_ID>1073741833</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1008</GENSTAMP>
+ <GENSTAMP>1009</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -706,31 +801,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>56</TXID>
- <BLOCK_ID>1073741833</BLOCK_ID>
+ <TXID>63</TXID>
+ <BLOCK_ID>1073741834</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>57</TXID>
- <GENSTAMPV2>1009</GENSTAMPV2>
+ <TXID>64</TXID>
+ <GENSTAMPV2>1010</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>58</TXID>
+ <TXID>65</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
- <BLOCK_ID>1073741832</BLOCK_ID>
+ <BLOCK_ID>1073741833</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1008</GENSTAMP>
+ <GENSTAMP>1009</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741833</BLOCK_ID>
+ <BLOCK_ID>1073741834</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1009</GENSTAMP>
+ <GENSTAMP>1010</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -739,23 +834,18 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>59</TXID>
+ <TXID>66</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380579</MTIME>
- <ATIME>1422406380560</ATIME>
+ <MTIME>1511309632643</MTIME>
+ <ATIME>1511309632620</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<BLOCK>
- <BLOCK_ID>1073741831</BLOCK_ID>
- <NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1007</GENSTAMP>
- </BLOCK>
- <BLOCK>
<BLOCK_ID>1073741832</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
<GENSTAMP>1008</GENSTAMP>
@@ -765,8 +855,13 @@
<NUM_BYTES>512</NUM_BYTES>
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
+ <BLOCK>
+ <BLOCK_ID>1073741834</BLOCK_ID>
+ <NUM_BYTES>512</NUM_BYTES>
+ <GENSTAMP>1010</GENSTAMP>
+ </BLOCK>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -775,64 +870,64 @@
<RECORD>
<OPCODE>OP_CONCAT_DELETE</OPCODE>
<DATA>
- <TXID>60</TXID>
+ <TXID>67</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
- <TIMESTAMP>1422406380582</TIMESTAMP>
+ <TIMESTAMP>1511309632648</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>57</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>63</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>61</TXID>
+ <TXID>68</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16392</INODEID>
+ <INODEID>16393</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380586</MTIME>
- <ATIME>1422406380586</ATIME>
+ <MTIME>1511309632651</MTIME>
+ <ATIME>1511309632651</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>59</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>65</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>62</TXID>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <TXID>69</TXID>
+ <BLOCK_ID>1073741835</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>63</TXID>
- <GENSTAMPV2>1010</GENSTAMPV2>
+ <TXID>70</TXID>
+ <GENSTAMPV2>1011</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>64</TXID>
+ <TXID>71</TXID>
<PATH>/file_create</PATH>
<BLOCK>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <BLOCK_ID>1073741835</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1010</GENSTAMP>
+ <GENSTAMP>1011</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -841,31 +936,31 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>65</TXID>
- <BLOCK_ID>1073741835</BLOCK_ID>
+ <TXID>72</TXID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>66</TXID>
- <GENSTAMPV2>1011</GENSTAMPV2>
+ <TXID>73</TXID>
+ <GENSTAMPV2>1012</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>67</TXID>
+ <TXID>74</TXID>
<PATH>/file_create</PATH>
<BLOCK>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <BLOCK_ID>1073741835</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1010</GENSTAMP>
+ <GENSTAMP>1011</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741835</BLOCK_ID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1011</GENSTAMP>
+ <GENSTAMP>1012</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -874,29 +969,29 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>68</TXID>
+ <TXID>75</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380599</MTIME>
- <ATIME>1422406380586</ATIME>
+ <MTIME>1511309632667</MTIME>
+ <ATIME>1511309632651</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<BLOCK>
- <BLOCK_ID>1073741834</BLOCK_ID>
+ <BLOCK_ID>1073741835</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1010</GENSTAMP>
+ <GENSTAMP>1011</GENSTAMP>
</BLOCK>
<BLOCK>
- <BLOCK_ID>1073741835</BLOCK_ID>
+ <BLOCK_ID>1073741836</BLOCK_ID>
<NUM_BYTES>512</NUM_BYTES>
- <GENSTAMP>1011</GENSTAMP>
+ <GENSTAMP>1012</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -905,79 +1000,79 @@
<RECORD>
<OPCODE>OP_TRUNCATE</OPCODE>
<DATA>
- <TXID>69</TXID>
+ <TXID>76</TXID>
<SRC>/file_create</SRC>
- <CLIENTNAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENTNAME>
+ <CLIENTNAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENTNAME>
<CLIENTMACHINE>127.0.0.1</CLIENTMACHINE>
<NEWLENGTH>512</NEWLENGTH>
- <TIMESTAMP>1422406380601</TIMESTAMP>
+ <TIMESTAMP>1511309632671</TIMESTAMP>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SYMLINK</OPCODE>
<DATA>
- <TXID>70</TXID>
+ <TXID>77</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16393</INODEID>
+ <INODEID>16394</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
- <MTIME>1422406380606</MTIME>
- <ATIME>1422406380606</ATIME>
+ <MTIME>1511309632686</MTIME>
+ <ATIME>1511309632686</ATIME>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>66</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>72</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>71</TXID>
+ <TXID>78</TXID>
<LENGTH>0</LENGTH>
- <INODEID>16394</INODEID>
+ <INODEID>16395</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406380608</MTIME>
- <ATIME>1422406380608</ATIME>
+ <MTIME>1511309632689</MTIME>
+ <ATIME>1511309632689</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_2134933941_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>67</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>73</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>72</TXID>
- <BLOCK_ID>1073741836</BLOCK_ID>
+ <TXID>79</TXID>
+ <BLOCK_ID>1073741837</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>73</TXID>
- <GENSTAMPV2>1012</GENSTAMPV2>
+ <TXID>80</TXID>
+ <GENSTAMPV2>1013</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>74</TXID>
+ <TXID>81</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
- <BLOCK_ID>1073741836</BLOCK_ID>
+ <BLOCK_ID>1073741837</BLOCK_ID>
<NUM_BYTES>0</NUM_BYTES>
- <GENSTAMP>1012</GENSTAMP>
+ <GENSTAMP>1013</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -986,12 +1081,12 @@
<RECORD>
<OPCODE>OP_UPDATE_BLOCKS</OPCODE>
<DATA>
- <TXID>75</TXID>
+ <TXID>82</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
- <BLOCK_ID>1073741836</BLOCK_ID>
+ <BLOCK_ID>1073741837</BLOCK_ID>
<NUM_BYTES>11</NUM_BYTES>
- <GENSTAMP>1012</GENSTAMP>
+ <GENSTAMP>1013</GENSTAMP>
</BLOCK>
<RPC_CLIENTID/>
<RPC_CALLID>-2</RPC_CALLID>
@@ -1000,40 +1095,40 @@
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>76</TXID>
- <GENSTAMPV2>1013</GENSTAMPV2>
+ <TXID>83</TXID>
+ <GENSTAMPV2>1014</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
- <TXID>77</TXID>
- <LEASEHOLDER>DFSClient_NONMAPREDUCE_-156773767_1</LEASEHOLDER>
+ <TXID>84</TXID>
+ <LEASEHOLDER>DFSClient_NONMAPREDUCE_2134933941_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
- <NEWHOLDER>HDFS_NameNode</NEWHOLDER>
+ <NEWHOLDER>HDFS_NameNode-2017-11-21 16:13:54,700-0800</NEWHOLDER>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>78</TXID>
+ <TXID>85</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1422406383261</MTIME>
- <ATIME>1422406380608</ATIME>
+ <MTIME>1511309634880</MTIME>
+ <ATIME>1511309632689</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME/>
<CLIENT_MACHINE/>
<OVERWRITE>false</OVERWRITE>
<BLOCK>
- <BLOCK_ID>1073741836</BLOCK_ID>
+ <BLOCK_ID>1073741837</BLOCK_ID>
<NUM_BYTES>11</NUM_BYTES>
- <GENSTAMP>1013</GENSTAMP>
+ <GENSTAMP>1014</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>xyao</USERNAME>
+ <USERNAME>lei</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -1042,72 +1137,73 @@
<RECORD>
<OPCODE>OP_ADD_CACHE_POOL</OPCODE>
<DATA>
- <TXID>79</TXID>
+ <TXID>86</TXID>
<POOLNAME>pool1</POOLNAME>
- <OWNERNAME>xyao</OWNERNAME>
+ <OWNERNAME>lei</OWNERNAME>
<GROUPNAME>staff</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>74</RPC_CALLID>
+ <DEFAULTREPLICATION>1</DEFAULTREPLICATION>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>80</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MODIFY_CACHE_POOL</OPCODE>
<DATA>
- <TXID>80</TXID>
+ <TXID>87</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>75</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>81</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>81</TXID>
+ <TXID>88</TXID>
<ID>1</ID>
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
- <EXPIRATION>2305844431620077592</EXPIRATION>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>76</RPC_CALLID>
+ <EXPIRATION>2305844520523329692</EXPIRATION>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>82</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MODIFY_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>82</TXID>
+ <TXID>89</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>77</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>83</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>83</TXID>
+ <TXID>90</TXID>
<ID>1</ID>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>78</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>84</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
<DATA>
- <TXID>84</TXID>
+ <TXID>91</TXID>
<POOLNAME>pool1</POOLNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>79</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>85</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_ACL</OPCODE>
<DATA>
- <TXID>85</TXID>
+ <TXID>92</TXID>
<SRC>/file_concat_target</SRC>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
@@ -1140,61 +1236,61 @@
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>86</TXID>
+ <TXID>93</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>81</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>87</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>87</TXID>
+ <TXID>94</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
<VALUE>0x373839</VALUE>
</XATTR>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>82</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>88</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_XATTR</OPCODE>
<DATA>
- <TXID>88</TXID>
+ <TXID>95</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
</XATTR>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>83</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>89</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>89</TXID>
+ <TXID>96</TXID>
<CODEC>rs</CODEC>
<DATAUNITS>3</DATAUNITS>
<PARITYUNITS>2</PARITYUNITS>
<CELLSIZE>8192</CELLSIZE>
<EXTRAOPTIONS>0</EXTRAOPTIONS>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>84</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>90</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>90</TXID>
+ <TXID>97</TXID>
<CODEC>rs</CODEC>
<DATAUNITS>6</DATAUNITS>
<PARITYUNITS>10</PARITYUNITS>
@@ -1204,82 +1300,82 @@
<KEY>dummyKey</KEY>
<VALUE>dummyValue</VALUE>
</EXTRAOPTION>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>85</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>91</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ENABLE_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>91</TXID>
+ <TXID>98</TXID>
<POLICYNAME>RS-3-2-8k</POLICYNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>86</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>92</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ENABLE_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>92</TXID>
+ <TXID>99</TXID>
<POLICYNAME>RS-6-10-4k</POLICYNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>87</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>93</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_DISABLE_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>93</TXID>
+ <TXID>100</TXID>
<POLICYNAME>RS-3-2-8k</POLICYNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>88</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>94</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_DISABLE_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>94</TXID>
+ <TXID>101</TXID>
<POLICYNAME>RS-6-10-4k</POLICYNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>89</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>95</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>95</TXID>
+ <TXID>102</TXID>
<POLICYNAME>RS-3-2-8k</POLICYNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>90</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>96</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_ERASURE_CODING_POLICY</OPCODE>
<DATA>
- <TXID>96</TXID>
+ <TXID>103</TXID>
<POLICYNAME>RS-6-10-4k</POLICYNAME>
- <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
- <RPC_CALLID>91</RPC_CALLID>
+ <RPC_CLIENTID>a4dc081c-6d6f-42d6-af5b-d260228f1aad</RPC_CLIENTID>
+ <RPC_CALLID>97</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
- <TXID>97</TXID>
- <STARTTIME>1422406383706</STARTTIME>
+ <TXID>104</TXID>
+ <STARTTIME>1511309635904</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
- <TXID>98</TXID>
- <FINALIZETIME>1422406383706</FINALIZETIME>
+ <TXID>105</TXID>
+ <FINALIZETIME>1511309635904</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_END_LOG_SEGMENT</OPCODE>
<DATA>
- <TXID>99</TXID>
+ <TXID>106</TXID>
</DATA>
</RECORD>
</EDITS>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/49] hadoop git commit: YARN-7489. ConcurrentModificationException
in RMAppImpl#getRMAppMetrics. Contributed by Tao Yang.
Posted by su...@apache.org.
YARN-7489. ConcurrentModificationException in RMAppImpl#getRMAppMetrics. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5b81a4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5b81a4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5b81a4f
Branch: refs/heads/YARN-5881
Commit: b5b81a4f086126974c75dab9b54aea20a0c1a48f
Parents: 96e6a99
Author: bibinchundatt <bi...@apache.org>
Authored: Sat Nov 18 19:25:29 2017 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Sat Nov 18 19:25:29 2017 +0530
----------------------------------------------------------------------
.../server/resourcemanager/rmapp/RMAppImpl.java | 58 +++++++++++---------
1 file changed, 31 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5b81a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 85d355f..6896254 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1641,35 +1641,39 @@ public class RMAppImpl implements RMApp, Recoverable {
int numNonAMContainerPreempted = 0;
Map<String, Long> resourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedSecondsMap = new HashMap<>();
-
- for (RMAppAttempt attempt : attempts.values()) {
- if (null != attempt) {
- RMAppAttemptMetrics attemptMetrics =
- attempt.getRMAppAttemptMetrics();
- Resources.addTo(resourcePreempted,
- attemptMetrics.getResourcePreempted());
- numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0;
- numNonAMContainerPreempted +=
- attemptMetrics.getNumNonAMContainersPreempted();
- // getAggregateAppResourceUsage() will calculate resource usage stats
- // for both running and finished containers.
- AggregateAppResourceUsage resUsage =
- attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
- for (Map.Entry<String, Long> entry : resUsage
- .getResourceUsageSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- resourceSecondsMap.put(entry.getKey(), value);
- }
- for (Map.Entry<String, Long> entry : attemptMetrics
- .getPreemptedResourceSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- preemptedSecondsMap.put(entry.getKey(), value);
+ this.readLock.lock();
+ try {
+ for (RMAppAttempt attempt : attempts.values()) {
+ if (null != attempt) {
+ RMAppAttemptMetrics attemptMetrics =
+ attempt.getRMAppAttemptMetrics();
+ Resources.addTo(resourcePreempted,
+ attemptMetrics.getResourcePreempted());
+ numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0;
+ numNonAMContainerPreempted +=
+ attemptMetrics.getNumNonAMContainersPreempted();
+ // getAggregateAppResourceUsage() will calculate resource usage stats
+ // for both running and finished containers.
+ AggregateAppResourceUsage resUsage =
+ attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
+ for (Map.Entry<String, Long> entry : resUsage
+ .getResourceUsageSecondsMap().entrySet()) {
+ long value = RMServerUtils
+ .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
+ value += entry.getValue();
+ resourceSecondsMap.put(entry.getKey(), value);
+ }
+ for (Map.Entry<String, Long> entry : attemptMetrics
+ .getPreemptedResourceSecondsMap().entrySet()) {
+ long value = RMServerUtils
+ .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
+ value += entry.getValue();
+ preemptedSecondsMap.put(entry.getKey(), value);
+ }
}
}
+ } finally {
+ this.readLock.unlock();
}
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/49] hadoop git commit: YARN-5534. Allow user provided Docker
volume mount list. (Contributed by Shane Kumpf)
Posted by su...@apache.org.
YARN-5534. Allow user provided Docker volume mount list. (Contributed by Shane Kumpf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d42a336c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d42a336c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d42a336c
Branch: refs/heads/YARN-5881
Commit: d42a336cfab106d052aa30d80d9d30904123cb55
Parents: de8b6ca
Author: Eric Yang <ey...@apache.org>
Authored: Wed Nov 22 13:05:34 2017 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Nov 22 13:05:34 2017 -0500
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 42 +++++++
.../linux/runtime/docker/DockerRunCommand.java | 12 ++
.../runtime/TestDockerContainerRuntime.java | 109 +++++++++++++++++++
.../src/site/markdown/DockerContainers.md | 48 ++++++++
4 files changed, 211 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d42a336c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 75a28e6..e61dc23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -65,6 +65,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*;
@@ -134,6 +135,16 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
* source is an absolute path that is not a symlink and that points to a
* localized resource.
* </li>
+ * <li>
+ * {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify
+ + additional volume mounts for the Docker container. The value of the
+ * environment variable should be a comma-separated list of mounts.
+ * All such mounts must be given as {@code source:dest:mode}, and the mode
+ * must be "ro" (read-only) or "rw" (read-write) to specify the type of
+ * access being requested. The requested mounts will be validated by
+ * container-executor based on the values set in container-executor.cfg for
+ * {@code docker.allowed.ro-mounts} and {@code docker.allowed.rw-mounts}.
+ * </li>
* </ul>
*/
@InterfaceAudience.Private
@@ -151,6 +162,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
"^[a-zA-Z0-9][a-zA-Z0-9_.-]+$";
private static final Pattern hostnamePattern = Pattern.compile(
HOSTNAME_PATTERN);
+ private static final Pattern USER_MOUNT_PATTERN = Pattern.compile(
+ "(?<=^|,)([^:\\x00]+):([^:\\x00]+):([a-z]+)");
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_IMAGE =
@@ -176,6 +189,9 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
@InterfaceAudience.Private
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
+ @InterfaceAudience.Private
+ public static final String ENV_DOCKER_CONTAINER_MOUNTS =
+ "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS";
private Configuration conf;
private Context nmContext;
@@ -675,6 +691,32 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
}
}
+ if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) {
+ Matcher parsedMounts = USER_MOUNT_PATTERN.matcher(
+ environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
+ if (!parsedMounts.find()) {
+ throw new ContainerExecutionException(
+ "Unable to parse user supplied mount list: "
+ + environment.get(ENV_DOCKER_CONTAINER_MOUNTS));
+ }
+ parsedMounts.reset();
+ while (parsedMounts.find()) {
+ String src = parsedMounts.group(1);
+ String dst = parsedMounts.group(2);
+ String mode = parsedMounts.group(3);
+ if (!mode.equals("ro") && !mode.equals("rw")) {
+ throw new ContainerExecutionException(
+ "Invalid mount mode requested for mount: "
+ + parsedMounts.group());
+ }
+ if (mode.equals("ro")) {
+ runCommand.addReadOnlyMountLocation(src, dst);
+ } else {
+ runCommand.addReadWriteMountLocation(src, dst);
+ }
+ }
+ }
+
if (allowPrivilegedContainerExecution(container)) {
runCommand.setPrivileged();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d42a336c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
index 8734ba6..aa7d4d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -66,6 +66,12 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand addReadWriteMountLocation(String sourcePath, String
+ destinationPath) {
+ super.addCommandArguments("rw-mounts", sourcePath + ":" + destinationPath);
+ return this;
+ }
+
public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
destinationPath, boolean createSource) {
boolean sourceExists = new File(sourcePath).exists();
@@ -76,6 +82,12 @@ public class DockerRunCommand extends DockerCommand {
return this;
}
+ public DockerRunCommand addReadOnlyMountLocation(String sourcePath, String
+ destinationPath) {
+ super.addCommandArguments("ro-mounts", sourcePath + ":" + destinationPath);
+ return this;
+ }
+
public DockerRunCommand setVolumeDriver(String volumeDriver) {
super.addCommandArguments("volume-driver", volumeDriver);
return this;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d42a336c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 76aca04..6135493 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -1036,6 +1036,115 @@ public class TestDockerContainerRuntime {
}
@Test
+ public void testUserMounts()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException{
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, null);
+
+ env.put(
+ DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
+ "/tmp/foo:/tmp/foo:ro,/tmp/bar:/tmp/bar:rw");
+
+ runtime.launchContainer(builder.build());
+ PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
+ List<String> args = op.getArguments();
+ String dockerCommandFile = args.get(11);
+
+ List<String> dockerCommands = Files.readAllLines(
+ Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
+
+ Assert.assertEquals(14, dockerCommands.size());
+ Assert.assertEquals("[docker-command-execution]", dockerCommands.get(0));
+ Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE",
+ dockerCommands.get(1));
+ Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(2));
+ Assert.assertEquals(" detach=true", dockerCommands.get(3));
+ Assert.assertEquals(" docker-command=run", dockerCommands.get(4));
+ Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5));
+ Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6));
+ Assert.assertEquals(
+ " launch-command=bash,/test_container_work_dir/launch_container.sh",
+ dockerCommands.get(7));
+ Assert.assertEquals(" name=container_id", dockerCommands.get(8));
+ Assert.assertEquals(" net=host", dockerCommands.get(9));
+ Assert.assertEquals(" ro-mounts=/tmp/foo:/tmp/foo",
+ dockerCommands.get(10));
+ Assert.assertEquals(
+ " rw-mounts=/test_container_local_dir:/test_container_local_dir,"
+ + "/test_filecache_dir:/test_filecache_dir,"
+ + "/test_container_work_dir:/test_container_work_dir,"
+ + "/test_container_log_dir:/test_container_log_dir,"
+ + "/test_user_local_dir:/test_user_local_dir,"
+ + "/tmp/bar:/tmp/bar",
+ dockerCommands.get(11));
+ Assert.assertEquals(" user=run_as_user", dockerCommands.get(12));
+ Assert.assertEquals(" workdir=/test_container_work_dir",
+ dockerCommands.get(13));
+ }
+
+ @Test
+ public void testUserMountInvalid()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException{
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, null);
+
+ env.put(
+ DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
+ "source:target");
+
+ try {
+ runtime.launchContainer(builder.build());
+ Assert.fail("Expected a launch container failure due to invalid mount.");
+ } catch (ContainerExecutionException e) {
+ LOG.info("Caught expected exception : " + e);
+ }
+ }
+
+ @Test
+ public void testUserMountModeInvalid()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException{
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, null);
+
+ env.put(
+ DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
+ "source:target:other");
+
+ try {
+ runtime.launchContainer(builder.build());
+ Assert.fail("Expected a launch container failure due to invalid mode.");
+ } catch (ContainerExecutionException e) {
+ LOG.info("Caught expected exception : " + e);
+ }
+ }
+
+ @Test
+ public void testUserMountModeNulInvalid()
+ throws ContainerExecutionException, PrivilegedOperationException,
+ IOException{
+ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(
+ mockExecutor, mockCGroupsHandler);
+ runtime.initialize(conf, null);
+
+ env.put(
+ DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS,
+ "s\0ource:target:ro");
+
+ try {
+ runtime.launchContainer(builder.build());
+ Assert.fail("Expected a launch container failure due to NUL in mount.");
+ } catch (ContainerExecutionException e) {
+ LOG.info("Caught expected exception : " + e);
+ }
+ }
+
+ @Test
public void testContainerLivelinessCheck()
throws ContainerExecutionException, PrivilegedOperationException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d42a336c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
index dbbce7f..1a50c92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md
@@ -290,6 +290,7 @@ environment variables in the application's environment:
| `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. |
| `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. |
| `YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest", where the source is an absolute path that is not a symlink and that points to a localized resource. Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. |
+| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode" and the mode must be "ro" (read-only) or "rw" (read-write) to specify the type of access being requested. The requested mounts will be validated by container-executor based on the values set in container-executor.cfg for docker.allowed.ro-mounts and docker.allowed.rw-mounts. |
The first two are required. The remainder can be set as needed. While
controlling the container type through environment variables is somewhat less
@@ -302,6 +303,53 @@ the application will behave exactly as any other YARN application. Logs will be
aggregated and stored in the relevant history server. The application life cycle
will be the same as for a non-Docker application.
+Using Docker Bind Mounted Volumes
+---------------------------------
+
+**WARNING** Care should be taken when enabling this feature. Enabling access to
+directories such as, but not limited to, /, /etc, /run, or /home is not
+advisable and can result in containers negatively impacting the host or leaking
+sensitive information. **WARNING**
+
+Files and directories from the host are commonly needed within the Docker
+containers, which Docker provides through
+[volumes](https://docs.docker.com/engine/tutorials/dockervolumes/).
+Examples include localized resources, Apache Hadoop binaries, and sockets. To
+facilitate this need, YARN-6623 added the ability for administrators to set a
+whitelist of host directories that are allowed to be bind mounted as volumes
+into containers. YARN-5534 added the ability for users to supply a list of
+mounts that will be mounted into the containers, if allowed by the
+administrative whitelist.
+
+In order to make use of this feature, the following must be configured.
+
+* The administrator must define the volume whitelist in container-executor.cfg by setting `docker.allowed.ro-mounts` and `docker.allowed.rw-mounts` to the list of parent directories that are allowed to be mounted.
+* The application submitter requests the required volumes at application submission time using the `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` environment variable.
+
+The administrator supplied whitelist is defined as a comma separated list of
+directories that are allowed to be mounted into containers. The source directory
+supplied by the user must either match or be a child of the specified
+directory.
+
+The user supplied mount list is defined as a comma separated list in the form
+*source*:*destination*:*mode*. The source is the file or directory on the host.
+The destination is the path within the contatiner where the source will be bind
+mounted. The mode defines the mode the user expects for the mount, which can be
+ro (read-only) or rw (read-write).
+
+The following example outlines how to use this feature to mount the commonly
+needed /sys/fs/cgroup directory into the container running on YARN.
+
+The administrator sets docker.allowed.ro-mounts in container-executor.cfg to
+"/sys/fs/cgroup". Applications can now request that "/sys/fs/cgroup" be mounted
+from the host into the container in read-only mode.
+
+At application submission time, the YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS
+environment variable can then be set to request this mount. In this example,
+the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro".
+The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also
+be valid given the example admin whitelist.
+
Connecting to a Secure Docker Repository
----------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/49] hadoop git commit: YARN-7544. Use
queue-path.capacity/maximum-capacity to specify absolute min/max resources.
(Sunil G via wangda)
Posted by su...@apache.org.
YARN-7544. Use queue-path.capacity/maximum-capacity to specify absolute min/max resources. (Sunil G via wangda)
Change-Id: I685341be213eee500f51e02f01c91def89391c17
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85fc1b51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85fc1b51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85fc1b51
Branch: refs/heads/YARN-5881
Commit: 85fc1b51786322dbb0690e2eb5303b0223b66f0d
Parents: 0614467
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 23 19:53:26 2017 -0800
Committer: Sunil G <su...@apache.org>
Committed: Fri Nov 24 15:37:55 2017 +0530
----------------------------------------------------------------------
.../CapacitySchedulerConfiguration.java | 79 +++++++++++++++-----
.../TestAbsoluteResourceConfiguration.java | 30 ++++----
2 files changed, 74 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85fc1b51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index f81e097..e47c8d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -329,7 +329,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores";
- public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)";
+ public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "^\\[[\\w\\.,\\-_=\\ /]+\\]$";
+
+ private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
public enum AbsoluteResourceType {
MEMORY, VCORES;
@@ -411,14 +413,29 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
public float getNonLabeledQueueCapacity(String queue) {
- float capacity = queue.equals("root") ? 100.0f : getFloat(
- getQueuePrefix(queue) + CAPACITY, 0f);
- if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
- throw new IllegalArgumentException("Illegal " +
- "capacity of " + capacity + " for queue " + queue);
+ String configuredCapacity = get(getQueuePrefix(queue) + CAPACITY);
+ boolean matcher = (configuredCapacity != null)
+ && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+ if (matcher) {
+ // Return capacity in percentage as 0 for non-root queues and 100 for
+ // root.From AbstractCSQueue, absolute resource will be parsed and
+ // updated. Once nodes are added/removed in cluster, capacity in
+ // percentage will also be re-calculated.
+ return queue.equals("root") ? 100.0f : 0f;
}
- LOG.debug("CSConf - getCapacity: queuePrefix=" + getQueuePrefix(queue) +
- ", capacity=" + capacity);
+
+ float capacity = queue.equals("root")
+ ? 100.0f
+ : (configuredCapacity == null)
+ ? 0f
+ : Float.parseFloat(configuredCapacity);
+ if (capacity < MINIMUM_CAPACITY_VALUE
+ || capacity > MAXIMUM_CAPACITY_VALUE) {
+ throw new IllegalArgumentException(
+ "Illegal " + "capacity of " + capacity + " for queue " + queue);
+ }
+ LOG.debug("CSConf - getCapacity: queuePrefix=" + getQueuePrefix(queue)
+ + ", capacity=" + capacity);
return capacity;
}
@@ -433,10 +450,23 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
public float getNonLabeledQueueMaximumCapacity(String queue) {
- float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY,
- MAXIMUM_CAPACITY_VALUE);
- maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ?
- MAXIMUM_CAPACITY_VALUE : maxCapacity;
+ String configuredCapacity = get(getQueuePrefix(queue) + MAXIMUM_CAPACITY);
+ boolean matcher = (configuredCapacity != null)
+ && RESOURCE_PATTERN.matcher(configuredCapacity).find();
+ if (matcher) {
+ // Return capacity in percentage as 0 for non-root queues and 100 for
+ // root.From AbstractCSQueue, absolute resource will be parsed and
+ // updated. Once nodes are added/removed in cluster, capacity in
+ // percentage will also be re-calculated.
+ return 100.0f;
+ }
+
+ float maxCapacity = (configuredCapacity == null)
+ ? MAXIMUM_CAPACITY_VALUE
+ : Float.parseFloat(configuredCapacity);
+ maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE)
+ ? MAXIMUM_CAPACITY_VALUE
+ : maxCapacity;
return maxCapacity;
}
@@ -590,6 +620,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
private float internalGetLabeledQueueCapacity(String queue, String label, String suffix,
float defaultValue) {
String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
+ boolean matcher = (capacityPropertyName != null)
+ && RESOURCE_PATTERN.matcher(capacityPropertyName).find();
+ if (matcher) {
+ // Return capacity in percentage as 0 for non-root queues and 100 for
+ // root.From AbstractCSQueue, absolute resource will be parsed and
+ // updated. Once nodes are added/removed in cluster, capacity in
+ // percentage will also be re-calculated.
+ return defaultValue;
+ }
+
float capacity = getFloat(capacityPropertyName, defaultValue);
if (capacity < MINIMUM_CAPACITY_VALUE
|| capacity > MAXIMUM_CAPACITY_VALUE) {
@@ -1722,7 +1762,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public Resource getMinimumResourceRequirement(String label, String queue,
Set<String> resourceTypes) {
return internalGetLabeledResourceRequirementForQueue(queue, label,
- resourceTypes, MINIMUM_RESOURCE);
+ resourceTypes, CAPACITY);
}
/**
@@ -1739,19 +1779,19 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public Resource getMaximumResourceRequirement(String label, String queue,
Set<String> resourceTypes) {
return internalGetLabeledResourceRequirementForQueue(queue, label,
- resourceTypes, MAXIMUM_RESOURCE);
+ resourceTypes, MAXIMUM_CAPACITY);
}
@VisibleForTesting
public void setMinimumResourceRequirement(String label, String queue,
Resource resource) {
- updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE);
+ updateMinMaxResourceToConf(label, queue, resource, CAPACITY);
}
@VisibleForTesting
public void setMaximumResourceRequirement(String label, String queue,
Resource resource) {
- updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE);
+ updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_CAPACITY);
}
private void updateMinMaxResourceToConf(String label, String queue,
@@ -1786,8 +1826,8 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
// Define resource here.
Resource resource = Resource.newInstance(0l, 0);
- Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE)
- .matcher(resourceString);
+ Matcher matcher = RESOURCE_PATTERN.matcher(resourceString);
+
/*
* Absolute resource configuration for a queue will be grouped by "[]".
* Syntax of absolute resource config could be like below
@@ -1795,11 +1835,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
*/
if (matcher.find()) {
// Get the sub-group.
- String subGroup = matcher.group(1);
+ String subGroup = matcher.group(0);
if (subGroup.trim().isEmpty()) {
return Resources.none();
}
+ subGroup = subGroup.substring(1, subGroup.length() - 1);
for (String kvPair : subGroup.trim().split(",")) {
String[] splits = kvPair.split("=");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85fc1b51/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
index 5a66281..80d16bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
@@ -233,7 +232,12 @@ public class TestAbsoluteResourceConfiguration {
@Test
public void testSimpleValidateAbsoluteResourceConfig() throws Exception {
/**
- * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ * Queue structure is as follows.
+ * root
+ * / | \
+ * a b c
+ * / \ |
+ * a1 a2 b1
*
* Test below cases 1) Configure percentage based capacity and absolute
* resource together. 2) As per above tree structure, ensure all values
@@ -258,21 +262,15 @@ public class TestAbsoluteResourceConfiguration {
// Get queue object to verify min/max resource configuration.
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
- // 1. Create a new config with capcity and min/max together. Ensure an
- // exception is thrown.
+ // 1. Create a new config with min/max.
CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration(
true);
setupMinMaxResourceConfiguration(csConf1);
try {
cs.reinitialize(csConf1, rm.getRMContext());
- Assert.fail();
} catch (IOException e) {
- Assert.assertTrue(e instanceof IOException);
- Assert.assertEquals(
- "Failed to re-init queues : Queue 'queueA' should use either"
- + " percentage based capacity configuration or absolute resource.",
- e.getMessage());
+ Assert.fail();
}
rm.stop();
@@ -368,7 +366,12 @@ public class TestAbsoluteResourceConfiguration {
@Test
public void testComplexValidateAbsoluteResourceConfig() throws Exception {
/**
- * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1
+ * Queue structure is as follows.
+ * root
+ * / | \
+ * a b c
+ * / \ |
+ * a1 a2 b1
*
* Test below cases: 1) Parent and its child queues must use either
* percentage based or absolute resource configuration. 2) Parent's min
@@ -396,11 +399,6 @@ public class TestAbsoluteResourceConfiguration {
csConf.setCapacity(QUEUEB_FULL, 25f);
csConf.setCapacity(QUEUEC_FULL, 25f);
- // Also unset resource based config.
- csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none());
- csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none());
- csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none());
-
// Get queue object to verify min/max resource configuration.
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
index 230dbad..02236eb 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/scale/AbstractSTestS3AHugeFiles.java
@@ -26,6 +26,7 @@ import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressEventType;
import com.amazonaws.event.ProgressListener;
import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.s3a.S3ATestUtils;
import org.junit.FixMethodOrder;
import org.junit.Test;
import org.junit.runners.MethodSorters;
@@ -70,16 +71,26 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
private int uploadBlockSize = DEFAULT_UPLOAD_BLOCKSIZE;
private int partitionSize;
+ private long filesize;
@Override
public void setup() throws Exception {
super.setup();
- final Path testPath = getTestPath();
- scaleTestDir = new Path(testPath, "scale");
+ scaleTestDir = new Path(getTestPath(), getTestSuiteName());
hugefile = new Path(scaleTestDir, "hugefile");
hugefileRenamed = new Path(scaleTestDir, "hugefileRenamed");
+ filesize = getTestPropertyBytes(getConf(), KEY_HUGE_FILESIZE,
+ DEFAULT_HUGE_FILESIZE);
}
+ /**
+ * Get the name of this test suite, which is used in path generation.
+ * Base implementation uses {@link #getBlockOutputBufferName()} for this.
+ * @return the name of the suite.
+ */
+ public String getTestSuiteName() {
+ return getBlockOutputBufferName();
+ }
/**
* Note that this can get called before test setup.
@@ -88,7 +99,7 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
@Override
protected Configuration createScaleConfiguration() {
Configuration conf = super.createScaleConfiguration();
- partitionSize = (int)getTestPropertyBytes(conf,
+ partitionSize = (int) getTestPropertyBytes(conf,
KEY_HUGE_PARTITION_SIZE,
DEFAULT_PARTITION_SIZE);
assertTrue("Partition size too small: " + partitionSize,
@@ -99,6 +110,7 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
conf.setInt(MULTIPART_SIZE, partitionSize);
conf.set(USER_AGENT_PREFIX, "STestS3AHugeFileCreate");
conf.set(FAST_UPLOAD_BUFFER, getBlockOutputBufferName());
+ S3ATestUtils.disableFilesystemCaching(conf);
return conf;
}
@@ -111,17 +123,16 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
@Test
public void test_010_CreateHugeFile() throws IOException {
assertFalse("Please run this test sequentially to avoid timeouts" +
- " and bandwidth problems", isParallelExecution());
- long filesize = getTestPropertyBytes(getConf(), KEY_HUGE_FILESIZE,
- DEFAULT_HUGE_FILESIZE);
+ " and bandwidth problems", isParallelExecution());
long filesizeMB = filesize / _1MB;
// clean up from any previous attempts
deleteHugeFile();
+ Path fileToCreate = getPathOfFileToCreate();
describe("Creating file %s of size %d MB" +
" with partition size %d buffered by %s",
- hugefile, filesizeMB, partitionSize, getBlockOutputBufferName());
+ fileToCreate, filesizeMB, partitionSize, getBlockOutputBufferName());
// now do a check of available upload time, with a pessimistic bandwidth
// (that of remote upload tests). If the test times out then not only is
@@ -134,7 +145,7 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
assertTrue(String.format("Timeout set in %s seconds is too low;" +
" estimating upload time of %d seconds at 1 MB/s." +
" Rerun tests with -D%s=%d",
- timeout, uploadTime, KEY_TEST_TIMEOUT, uploadTime * 2),
+ timeout, uploadTime, KEY_TEST_TIMEOUT, uploadTime * 2),
uploadTime < timeout);
assertEquals("File size set in " + KEY_HUGE_FILESIZE + " = " + filesize
+ " is not a multiple of " + uploadBlockSize,
@@ -162,7 +173,7 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
S3AInstrumentation.OutputStreamStatistics streamStatistics;
long blocksPer10MB = blocksPerMB * 10;
ProgressCallback progress = new ProgressCallback(timer);
- try (FSDataOutputStream out = fs.create(hugefile,
+ try (FSDataOutputStream out = fs.create(fileToCreate,
true,
uploadBlockSize,
progress)) {
@@ -219,14 +230,8 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
toHuman(timer.nanosPerOperation(putRequestCount)));
assertEquals("active put requests in \n" + fs,
0, gaugeValue(putRequestsActive));
- ContractTestUtils.assertPathExists(fs, "Huge file", hugefile);
- FileStatus status = fs.getFileStatus(hugefile);
- ContractTestUtils.assertIsFile(hugefile, status);
- assertEquals("File size in " + status, filesize, status.getLen());
- if (progress != null) {
- progress.verifyNoFailures("Put file " + hugefile
- + " of size " + filesize);
- }
+ progress.verifyNoFailures(
+ "Put file " + fileToCreate + " of size " + filesize);
if (streamStatistics != null) {
assertEquals("actively allocated blocks in " + streamStatistics,
0, streamStatistics.blocksActivelyAllocated());
@@ -234,6 +239,39 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
}
/**
+ * Get the path of the file which is to created. This is normally
+ * {@link #hugefile}
+ * @return the path to use when creating the file.
+ */
+ protected Path getPathOfFileToCreate() {
+ return this.hugefile;
+ }
+
+ protected Path getScaleTestDir() {
+ return scaleTestDir;
+ }
+
+ protected Path getHugefile() {
+ return hugefile;
+ }
+
+ public void setHugefile(Path hugefile) {
+ this.hugefile = hugefile;
+ }
+
+ protected Path getHugefileRenamed() {
+ return hugefileRenamed;
+ }
+
+ protected int getUploadBlockSize() {
+ return uploadBlockSize;
+ }
+
+ protected int getPartitionSize() {
+ return partitionSize;
+ }
+
+ /**
* Progress callback from AWS. Likely to come in on a different thread.
*/
private final class ProgressCallback implements Progressable,
@@ -294,22 +332,54 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
}
private void verifyNoFailures(String operation) {
- assertEquals("Failures in " + operation +": " + this, 0, failures.get());
+ assertEquals("Failures in " + operation + ": " + this, 0, failures.get());
}
}
+ /**
+ * Assume that the huge file exists; skip the test if it does not.
+ * @throws IOException IO failure
+ */
void assumeHugeFileExists() throws IOException {
+ assumeFileExists(this.hugefile);
+ }
+
+ /**
+ * Assume a specific file exists.
+ * @param file file to look for
+ * @throws IOException IO problem
+ */
+ private void assumeFileExists(Path file) throws IOException {
S3AFileSystem fs = getFileSystem();
- ContractTestUtils.assertPathExists(fs, "huge file not created", hugefile);
- FileStatus status = fs.getFileStatus(hugefile);
- ContractTestUtils.assertIsFile(hugefile, status);
- assertTrue("File " + hugefile + " is empty", status.getLen() > 0);
+ ContractTestUtils.assertPathExists(fs, "huge file not created",
+ file);
+ FileStatus status = fs.getFileStatus(file);
+ ContractTestUtils.assertIsFile(file, status);
+ assertTrue("File " + file + " is empty", status.getLen() > 0);
}
private void logFSState() {
LOG.info("File System state after operation:\n{}", getFileSystem());
}
+ /**
+ * This is the set of actions to perform when verifying the file actually
+ * was created. With the s3guard committer, the file doesn't come into
+ * existence; a different set of assertions must be checked.
+ */
+ @Test
+ public void test_030_postCreationAssertions() throws Throwable {
+ S3AFileSystem fs = getFileSystem();
+ ContractTestUtils.assertPathExists(fs, "Huge file", hugefile);
+ FileStatus status = fs.getFileStatus(hugefile);
+ ContractTestUtils.assertIsFile(hugefile, status);
+ assertEquals("File size in " + status, filesize, status.getLen());
+ }
+
+ /**
+ * Read in the file using Positioned read(offset) calls.
+ * @throws Throwable failure
+ */
@Test
public void test_040_PositionedReadHugeFile() throws Throwable {
assumeHugeFileExists();
@@ -323,11 +393,11 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
describe("Positioned reads of %s %s", filetype, hugefile);
S3AFileSystem fs = getFileSystem();
FileStatus status = fs.getFileStatus(hugefile);
- long filesize = status.getLen();
+ long size = status.getLen();
int ops = 0;
final int bufferSize = 8192;
byte[] buffer = new byte[bufferSize];
- long eof = filesize - 1;
+ long eof = size - 1;
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
ContractTestUtils.NanoTimer readAtByte0, readAtByte0Again, readAtEOF;
@@ -348,23 +418,27 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
ops++;
LOG.info("Final stream state: {}", in);
}
- long mb = Math.max(filesize / _1MB, 1);
+ long mb = Math.max(size / _1MB, 1);
logFSState();
- timer.end("time to performed positioned reads of %s of %d MB ",
+ timer.end("time to perform positioned reads of %s of %d MB ",
filetype, mb);
LOG.info("Time per positioned read = {} nS",
toHuman(timer.nanosPerOperation(ops)));
}
+ /**
+ * Read in the entire file using read() calls.
+ * @throws Throwable failure
+ */
@Test
public void test_050_readHugeFile() throws Throwable {
assumeHugeFileExists();
describe("Reading %s", hugefile);
S3AFileSystem fs = getFileSystem();
FileStatus status = fs.getFileStatus(hugefile);
- long filesize = status.getLen();
- long blocks = filesize / uploadBlockSize;
+ long size = status.getLen();
+ long blocks = size / uploadBlockSize;
byte[] data = new byte[uploadBlockSize];
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
@@ -375,11 +449,11 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
LOG.info("Final stream state: {}", in);
}
- long mb = Math.max(filesize / _1MB, 1);
+ long mb = Math.max(size / _1MB, 1);
timer.end("time to read file of %d MB ", mb);
LOG.info("Time per MB to read = {} nS",
toHuman(timer.nanosPerOperation(mb)));
- bandwidth(timer, filesize);
+ bandwidth(timer, size);
logFSState();
}
@@ -389,18 +463,18 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
describe("renaming %s to %s", hugefile, hugefileRenamed);
S3AFileSystem fs = getFileSystem();
FileStatus status = fs.getFileStatus(hugefile);
- long filesize = status.getLen();
+ long size = status.getLen();
fs.delete(hugefileRenamed, false);
ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
fs.rename(hugefile, hugefileRenamed);
- long mb = Math.max(filesize / _1MB, 1);
+ long mb = Math.max(size / _1MB, 1);
timer.end("time to rename file of %d MB", mb);
LOG.info("Time per MB to rename = {} nS",
toHuman(timer.nanosPerOperation(mb)));
- bandwidth(timer, filesize);
+ bandwidth(timer, size);
logFSState();
FileStatus destFileStatus = fs.getFileStatus(hugefileRenamed);
- assertEquals(filesize, destFileStatus.getLen());
+ assertEquals(size, destFileStatus.getLen());
// rename back
ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
@@ -408,24 +482,49 @@ public abstract class AbstractSTestS3AHugeFiles extends S3AScaleTestBase {
timer2.end("Renaming back");
LOG.info("Time per MB to rename = {} nS",
toHuman(timer2.nanosPerOperation(mb)));
- bandwidth(timer2, filesize);
+ bandwidth(timer2, size);
}
+ /**
+ * Cleanup: delete the files.
+ */
@Test
- public void test_999_DeleteHugeFiles() throws IOException {
- deleteHugeFile();
- ContractTestUtils.NanoTimer timer2 = new ContractTestUtils.NanoTimer();
- S3AFileSystem fs = getFileSystem();
- fs.delete(hugefileRenamed, false);
- timer2.end("time to delete %s", hugefileRenamed);
- ContractTestUtils.rm(fs, getTestPath(), true, true);
+ public void test_800_DeleteHugeFiles() throws IOException {
+ try {
+ deleteHugeFile();
+ delete(hugefileRenamed, false);
+ } finally {
+ ContractTestUtils.rm(getFileSystem(), getTestPath(), true, false);
+ }
+ }
+
+ /**
+ * After all the work, dump the statistics.
+ */
+ @Test
+ public void test_900_dumpStats() {
+ StringBuilder sb = new StringBuilder();
+
+ getFileSystem().getStorageStatistics()
+ .forEach(kv -> sb.append(kv.toString()).append("\n"));
+
+ LOG.info("Statistics\n{}", sb);
}
protected void deleteHugeFile() throws IOException {
- describe("Deleting %s", hugefile);
- NanoTimer timer = new NanoTimer();
- getFileSystem().delete(hugefile, false);
- timer.end("time to delete %s", hugefile);
+ delete(hugefile, false);
+ }
+
+ /**
+ * Delete any file, time how long it took.
+ * @param path path to delete
+ * @param recursive recursive flag
+ */
+ protected void delete(Path path, boolean recursive) throws IOException {
+ describe("Deleting %s", path);
+ ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer();
+ getFileSystem().delete(path, recursive);
+ timer.end("time to delete %s", path);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
index e8200da..b68f559 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
+++ b/hadoop-tools/hadoop-aws/src/test/resources/core-site.xml
@@ -27,7 +27,7 @@
<name>hadoop.tmp.dir</name>
<value>target/build/test</value>
<description>A base for other temporary directories.</description>
- <final>true</final>
+ <final>false</final>
</property>
<property>
@@ -36,6 +36,13 @@
<description>The endpoint for s3a://landsat-pds URLs</description>
</property>
+ <property>
+ <name>fs.s3a.bucket.landsat-pds.multipart.purge</name>
+ <value>false</value>
+ <description>Don't try to purge uploads in the read-only bucket, as
+ it will only create log noise.</description>
+ </property>
+
<!-- Make sure S3Guard is disabled for read-only bucket tests. -->
<property>
<name>fs.s3a.bucket.landsat-pds.metadatastore.impl</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
index acbe7f1..383dec8 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
@@ -18,15 +18,50 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} [%t] %-5p %c{2} (%F:%M(%L)) - %m%n
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
+log4j.logger.org.apache.hadoop.util.GSet=WARN
+# MiniDFS clusters can be noisy
+log4j.logger.org.apache.hadoop.hdfs.server=ERROR
+log4j.logger.org.apache.hadoop.metrics2=WARN
+log4j.logger.org.apache.hadoop.net.NetworkTopology=WARN
+log4j.logger.org.apache.hadoop.util.JvmPauseMonitor=WARN
+log4j.logger.org.apache.hadoop.ipc=WARN
+log4j.logger.org.apache.hadoop.http=WARN
+log4j.logger.org.apache.hadoop.security.authentication.server.AuthenticationFilter=WARN
+log4j.logger.org.apache.hadoop.util.HostsFileReader=WARN
+log4j.logger.org.apache.commons.beanutils=WARN
+log4j.logger.org.apache.hadoop.hdfs.StateChange=WARN
+log4j.logger.BlockStateChange=WARN
+log4j.logger.org.apache.hadoop.hdfs.DFSUtil=WARN
+
+## YARN can be noisy too
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.scheduler=WARN
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager=WARN
+log4j.logger.org.apache.hadoop.yarn.event=WARN
+log4j.logger.org.apache.hadoop.yarn.util.ResourceCalculatorPlugin=ERROR
+log4j.logger.org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor=WARN
+log4j.logger.org.apache.hadoop.mapred.IndexCache=WARN
+log4j.logger.org.apache.hadoop.yarn.webapp.WebApps=WARN
+log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.security=WARN
+log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
+log4j.logger.org.apache.hadoop.security.token.delegation=WARN
+log4j.logger.org.apache.hadoop.mapred.ShuffleHandler=WARN
+log4j.logger.org.apache.hadoop.ipc.Server=WARN
+#log4j.logger.=WARN
+
# for debugging low level S3a operations, uncomment these lines
# Log all S3A classes
#log4j.logger.org.apache.hadoop.fs.s3a=DEBUG
-
+#log4j.logger.org.apache.hadoop.fs.s3a.S3AUtils=INFO
+#log4j.logger.org.apache.hadoop.fs.s3a.Listing=INFO
# Log S3Guard classes
#log4j.logger.org.apache.hadoop.fs.s3a.s3guard=DEBUG
-# Enable debug logging of AWS DynamoDB client
+# Log Committer classes
+#log4j.logger.org.apache.hadoop.fs.s3a.commit=DEBUG
+
+# Enable debug logging of AWS Dynamo client
#log4j.logger.com.amazonaws.services.dynamodbv2.AmazonDynamoDB=DEBUG
# Log all HTTP requests made; includes S3 interaction. This may
@@ -35,3 +70,6 @@ log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
# Turn on low level HTTP protocol debugging
#log4j.logger.com.amazonaws.thirdparty.apache.http=DEBUG
+
+log4j.logger.org.apache.hadoop.mapreduce.lib.output=DEBUG
+log4j.logger.org.apache.hadoop.fs.s3a.S3AStorageStatistics=INFO
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java
index eddff20..491bcf9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/JsonSerDeser.java
@@ -18,212 +18,47 @@
package org.apache.hadoop.registry.client.binding;
-import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.registry.client.exceptions.InvalidRecordException;
import org.apache.hadoop.registry.client.exceptions.NoRecordException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.hadoop.util.JsonSerialization;
-import java.io.DataOutputStream;
import java.io.EOFException;
-import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.InputStream;
/**
* Support for marshalling objects to and from JSON.
* <p>
- * It constructs an object mapper as an instance field.
- * and synchronizes access to those methods
- * which use the mapper
+ * This extends {@link JsonSerialization} with the notion
+ * of a marker field in the JSON file, with
+ * <ol>
+ * <li>a fail-fast check for it before even trying to parse.</li>
+ * <li>Specific IOException subclasses for a failure.</li>
+ * </ol>
+ * The rationale for this is not only to support different things in the,
+ * registry, but the fact that all ZK nodes have a size > 0 when examined.
+ *
* @param <T> Type to marshal.
*/
-@InterfaceAudience.Private()
+@InterfaceAudience.Private
@InterfaceStability.Evolving
-public class JsonSerDeser<T> {
+public class JsonSerDeser<T> extends JsonSerialization<T> {
- private static final Logger LOG = LoggerFactory.getLogger(JsonSerDeser.class);
private static final String UTF_8 = "UTF-8";
public static final String E_NO_DATA = "No data at path";
public static final String E_DATA_TOO_SHORT = "Data at path too short";
public static final String E_MISSING_MARKER_STRING =
"Missing marker string: ";
- private final Class<T> classType;
- private final ObjectMapper mapper;
-
/**
* Create an instance bound to a specific type
* @param classType class to marshall
*/
public JsonSerDeser(Class<T> classType) {
- Preconditions.checkArgument(classType != null, "null classType");
- this.classType = classType;
- this.mapper = new ObjectMapper();
- mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- }
-
- /**
- * Get the simple name of the class type to be marshalled
- * @return the name of the class being marshalled
- */
- public String getName() {
- return classType.getSimpleName();
- }
-
- /**
- * Convert from JSON
- *
- * @param json input
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonParseException If the input is not well-formatted
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- @SuppressWarnings("unchecked")
- public synchronized T fromJson(String json)
- throws IOException, JsonParseException, JsonMappingException {
- try {
- return mapper.readValue(json, classType);
- } catch (IOException e) {
- LOG.error("Exception while parsing json : " + e + "\n" + json, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param jsonFile input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonParseException If the input is not well-formatted
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- @SuppressWarnings("unchecked")
- public synchronized T fromFile(File jsonFile)
- throws IOException, JsonParseException, JsonMappingException {
- try {
- return mapper.readValue(jsonFile, classType);
- } catch (IOException e) {
- LOG.error("Exception while parsing json file {}: {}", jsonFile, e);
- throw e;
- }
- }
-
- /**
- * Convert from a JSON file
- * @param resource input file
- * @return the parsed JSON
- * @throws IOException IO problems
- * @throws JsonParseException If the input is not well-formatted
- * @throws JsonMappingException failure to map from the JSON to this class
- */
- @SuppressWarnings({"IOResourceOpenedButNotSafelyClosed"})
- public synchronized T fromResource(String resource)
- throws IOException, JsonParseException, JsonMappingException {
- InputStream resStream = null;
- try {
- resStream = this.getClass().getResourceAsStream(resource);
- if (resStream == null) {
- throw new FileNotFoundException(resource);
- }
- return mapper.readValue(resStream, classType);
- } catch (IOException e) {
- LOG.error("Exception while parsing json resource {}: {}", resource, e);
- throw e;
- } finally {
- IOUtils.closeStream(resStream);
- }
- }
-
- /**
- * clone by converting to JSON and back again.
- * This is much less efficient than any Java clone process.
- * @param instance instance to duplicate
- * @return a new instance
- * @throws IOException problems.
- */
- public T fromInstance(T instance) throws IOException {
- return fromJson(toJson(instance));
- }
-
- /**
- * Load from a Hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @return a loaded CD
- * @throws IOException IO problems
- * @throws EOFException if not enough bytes were read in
- * @throws JsonParseException parse problems
- * @throws JsonMappingException O/J mapping problems
- */
- public T load(FileSystem fs, Path path)
- throws IOException, JsonParseException, JsonMappingException {
- FileStatus status = fs.getFileStatus(path);
- long len = status.getLen();
- byte[] b = new byte[(int) len];
- FSDataInputStream dataInputStream = fs.open(path);
- int count = dataInputStream.read(b);
- if (count != len) {
- throw new EOFException(path.toString() + ": read finished prematurely");
- }
- return fromBytes(path.toString(), b);
- }
-
- /**
- * Save a cluster description to a hadoop filesystem
- * @param fs filesystem
- * @param path path
- * @param overwrite should any existing file be overwritten
- * @throws IOException IO exception
- */
- public void save(FileSystem fs, Path path, T instance,
- boolean overwrite) throws
- IOException {
- FSDataOutputStream dataOutputStream = fs.create(path, overwrite);
- writeJsonAsBytes(instance, dataOutputStream);
- }
-
- /**
- * Write the json as bytes -then close the file
- * @param dataOutputStream an outout stream that will always be closed
- * @throws IOException on any failure
- */
- private void writeJsonAsBytes(T instance,
- DataOutputStream dataOutputStream) throws IOException {
- try {
- byte[] b = toBytes(instance);
- dataOutputStream.write(b);
- } finally {
- dataOutputStream.close();
- }
- }
-
- /**
- * Convert JSON To bytes
- * @param instance instance to convert
- * @return a byte array
- * @throws IOException
- */
- public byte[] toBytes(T instance) throws IOException {
- String json = toJson(instance);
- return json.getBytes(UTF_8);
+ super(classType, false, false);
}
/**
@@ -233,9 +68,10 @@ public class JsonSerDeser<T> {
* @throws IOException all problems
* @throws EOFException not enough data
* @throws InvalidRecordException if the parsing failed -the record is invalid
+ * @throws NoRecordException if the data is not considered a record: either
+ * it is too short or it did not contain the marker string.
*/
- public T fromBytes(String path, byte[] bytes) throws IOException,
- InvalidRecordException {
+ public T fromBytes(String path, byte[] bytes) throws IOException {
return fromBytes(path, bytes, "");
}
@@ -258,7 +94,7 @@ public class JsonSerDeser<T> {
* it is too short or it did not contain the marker string.
*/
public T fromBytes(String path, byte[] bytes, String marker)
- throws IOException, NoRecordException, InvalidRecordException {
+ throws IOException {
int len = bytes.length;
if (len == 0 ) {
throw new NoRecordException(path, E_NO_DATA);
@@ -278,30 +114,4 @@ public class JsonSerDeser<T> {
}
}
- /**
- * Convert an instance to a JSON string
- * @param instance instance to convert
- * @return a JSON string description
- * @throws JsonProcessingException Json generation problems
- */
- public synchronized String toJson(T instance) throws JsonProcessingException {
- mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
- return mapper.writeValueAsString(instance);
- }
-
- /**
- * Convert an instance to a string form for output. This is a robust
- * operation which will convert any JSON-generating exceptions into
- * error text.
- * @param instance non-null instance
- * @return a JSON string
- */
- public String toString(T instance) {
- Preconditions.checkArgument(instance != null, "Null instance argument");
- try {
- return toJson(instance);
- } catch (JsonProcessingException e) {
- return "Failed to convert to a string: " + e;
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/49] hadoop git commit: YARN-7513. Remove the scheduler lock in
FSAppAttempt.getWeight() (Contributed by Wilfred Spiegelenburg)
Posted by su...@apache.org.
YARN-7513. Remove the scheduler lock in FSAppAttempt.getWeight() (Contributed by Wilfred Spiegelenburg)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03c311ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03c311ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03c311ea
Branch: refs/heads/YARN-5881
Commit: 03c311eae3ad591630a452921172a4406dbda181
Parents: 67bbbe1
Author: yufei <yu...@apache.org>
Authored: Tue Nov 21 10:33:34 2017 -0800
Committer: yufei <yu...@apache.org>
Committed: Tue Nov 21 10:33:34 2017 -0800
----------------------------------------------------------------------
.../resourcemanager/scheduler/fair/FSAppAttempt.java | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03c311ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 94991eb..e711229 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -1304,20 +1304,14 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
@Override
public float getWeight() {
- double weight = 1.0;
+ float weight = 1.0F;
if (scheduler.isSizeBasedWeight()) {
- scheduler.getSchedulerReadLock().lock();
-
- try {
- // Set weight based on current memory demand
- weight = Math.log1p(getDemand().getMemorySize()) / Math.log(2);
- } finally {
- scheduler.getSchedulerReadLock().unlock();
- }
+ // Set weight based on current memory demand
+ weight = (float)(Math.log1p(demand.getMemorySize()) / Math.log(2));
}
- return (float)weight * this.getPriority().getPriority();
+ return weight * appPriority.getPriority();
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/49] hadoop git commit: HADOOP-13786 Add S3A committer for
zero-rename commits to S3 endpoints. Contributed by Steve Loughran and Ryan
Blue.
Posted by su...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitterFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitterFactory.java
new file mode 100644
index 0000000..b3bcca1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/AbstractS3ACommitterFactory.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory;
+
+/**
+ * Dynamically create the output committer based on subclass type and settings.
+ */
+public abstract class AbstractS3ACommitterFactory
+ extends PathOutputCommitterFactory {
+ public static final Logger LOG = LoggerFactory.getLogger(
+ AbstractS3ACommitterFactory.class);
+
+ @Override
+ public PathOutputCommitter createOutputCommitter(Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ FileSystem fs = getDestinationFileSystem(outputPath, context);
+ PathOutputCommitter outputCommitter;
+ if (fs instanceof S3AFileSystem) {
+ outputCommitter = createTaskCommitter((S3AFileSystem)fs,
+ outputPath, context);
+ } else {
+ throw new PathCommitException(outputPath,
+ "Filesystem not supported by this committer");
+ }
+ LOG.info("Using Commmitter {} for {}",
+ outputCommitter,
+ outputPath);
+ return outputCommitter;
+ }
+
+ /**
+ * Get the destination filesystem, returning null if there is none.
+ * Code using this must explicitly or implicitly look for a null value
+ * in the response.
+ * @param outputPath output path
+ * @param context job/task context
+ * @return the destination filesystem, if it can be determined
+ * @throws IOException if the FS cannot be instantiated
+ */
+ protected FileSystem getDestinationFileSystem(Path outputPath,
+ JobContext context)
+ throws IOException {
+ return outputPath != null ?
+ FileSystem.get(outputPath.toUri(), context.getConfiguration())
+ : null;
+ }
+
+ /**
+ * Implementation point: create a task committer for a specific filesystem.
+ * @param fileSystem destination FS.
+ * @param outputPath final output path for work
+ * @param context task context
+ * @return a committer
+ * @throws IOException any problem, including the FS not supporting
+ * the desired committer
+ */
+ public abstract PathOutputCommitter createTaskCommitter(
+ S3AFileSystem fileSystem,
+ Path outputPath,
+ TaskAttemptContext context) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
new file mode 100644
index 0000000..03cfcba
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitConstants.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import static org.apache.hadoop.mapreduce.lib.output.PathOutputCommitterFactory.COMMITTER_FACTORY_SCHEME_PATTERN;
+
+/**
+ * Constants for working with committers.
+ */
+@SuppressWarnings("unused")
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public final class CommitConstants {
+
+ private CommitConstants() {
+ }
+
+ /**
+ * Path for "magic" writes: path and {@link #PENDING_SUFFIX} files:
+ * {@value}.
+ */
+ public static final String MAGIC = "__magic";
+
+ /**
+ * Marker of the start of a directory tree for calculating
+ * the final path names: {@value}.
+ */
+ public static final String BASE = "__base";
+
+ /**
+ * Suffix applied to pending commit metadata: {@value}.
+ */
+ public static final String PENDING_SUFFIX = ".pending";
+
+ /**
+ * Suffix applied to multiple pending commit metadata: {@value}.
+ */
+ public static final String PENDINGSET_SUFFIX = ".pendingset";
+
+ /**
+ * Flag to indicate whether support for the Magic committer is enabled
+ * in the filesystem.
+ * Value: {@value}.
+ */
+ public static final String MAGIC_COMMITTER_PREFIX
+ = "fs.s3a.committer.magic";
+
+ /**
+ * Flag to indicate whether support for the Magic committer is enabled
+ * in the filesystem.
+ * Value: {@value}.
+ */
+ public static final String MAGIC_COMMITTER_ENABLED
+ = MAGIC_COMMITTER_PREFIX + ".enabled";
+
+ /**
+ * Flag to indicate whether a stream is a magic output stream;
+ * returned in {@code StreamCapabilities}
+ * Value: {@value}.
+ */
+ public static final String STREAM_CAPABILITY_MAGIC_OUTPUT
+ = "s3a:magic.output.stream";
+
+ /**
+ * Flag to indicate that a store supports magic committers.
+ * returned in {@code StreamCapabilities}
+ * Value: {@value}.
+ */
+ public static final String STORE_CAPABILITY_MAGIC_COMMITTER
+ = "s3a:magic.committer";
+
+ /**
+ * Is the committer enabled by default? No.
+ */
+ public static final boolean DEFAULT_MAGIC_COMMITTER_ENABLED = false;
+
+ /**
+ * This is the "Pending" directory of the {@code FileOutputCommitter};
+ * data written here is, in that algorithm, renamed into place.
+ * Value: {@value}.
+ */
+ public static final String TEMPORARY = "_temporary";
+
+ /**
+ * Temp data which is not auto-committed: {@value}.
+ * Uses a different name from normal just to make clear it is different.
+ */
+ public static final String TEMP_DATA = "__temp-data";
+
+
+ /**
+ * Flag to trigger creation of a marker file on job completion.
+ */
+ public static final String CREATE_SUCCESSFUL_JOB_OUTPUT_DIR_MARKER
+ = "mapreduce.fileoutputcommitter.marksuccessfuljobs";
+
+ /**
+ * Marker file to create on success: {@value}.
+ */
+ public static final String _SUCCESS = "_SUCCESS";
+
+ /** Default job marker option: {@value}. */
+ public static final boolean DEFAULT_CREATE_SUCCESSFUL_JOB_DIR_MARKER = true;
+
+ /**
+ * Key to set for the S3A schema to use the specific committer.
+ */
+ public static final String S3A_COMMITTER_FACTORY_KEY = String.format(
+ COMMITTER_FACTORY_SCHEME_PATTERN, "s3a");
+
+ /**
+ * S3 Committer factory: {@value}.
+ * This uses the value of {@link #FS_S3A_COMMITTER_NAME}
+ * to choose the final committer.
+ */
+ public static final String S3A_COMMITTER_FACTORY =
+ S3ACommitterFactory.CLASSNAME;
+
+ /**
+ * Option to identify the S3A committer:
+ * {@value}.
+ */
+ public static final String FS_S3A_COMMITTER_NAME =
+ "fs.s3a.committer.name";
+
+ /**
+ * Option for {@link #FS_S3A_COMMITTER_NAME}:
+ * classic/file output committer: {@value}.
+ */
+ public static final String COMMITTER_NAME_FILE = "file";
+
+ /**
+ * Option for {@link #FS_S3A_COMMITTER_NAME}:
+ * magic output committer: {@value}.
+ */
+ public static final String COMMITTER_NAME_MAGIC = "magic";
+
+ /**
+ * Option for {@link #FS_S3A_COMMITTER_NAME}:
+ * directory output committer: {@value}.
+ */
+ public static final String COMMITTER_NAME_DIRECTORY = "directory";
+
+ /**
+ * Option for {@link #FS_S3A_COMMITTER_NAME}:
+ * partition output committer: {@value}.
+ */
+ public static final String COMMITTER_NAME_PARTITIONED = "partitioned";
+
+ /**
+ * Option for final files to have a uniqueness name through job attempt info,
+ * falling back to a new UUID if there is no job attempt information to use.
+ * {@value}.
+ * When writing data with the "append" conflict option, this guarantees
+ * that new data will not overwrite any existing data.
+ */
+ public static final String FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES =
+ "fs.s3a.committer.staging.unique-filenames";
+ /**
+ * Default value for {@link #FS_S3A_COMMITTER_STAGING_UNIQUE_FILENAMES}:
+ * {@value}.
+ */
+ public static final boolean DEFAULT_STAGING_COMMITTER_UNIQUE_FILENAMES = true;
+
+ /**
+ * Staging committer conflict resolution policy: {@value}.
+ * Supported: fail, append, replace.
+ */
+ public static final String FS_S3A_COMMITTER_STAGING_CONFLICT_MODE =
+ "fs.s3a.committer.staging.conflict-mode";
+
+ /** Conflict mode: {@value}. */
+ public static final String CONFLICT_MODE_FAIL = "fail";
+
+ /** Conflict mode: {@value}. */
+ public static final String CONFLICT_MODE_APPEND = "append";
+
+ /** Conflict mode: {@value}. */
+ public static final String CONFLICT_MODE_REPLACE = "replace";
+
+ /** Default conflict mode: {@value}. */
+ public static final String DEFAULT_CONFLICT_MODE = CONFLICT_MODE_FAIL;
+
+ /**
+ * Number of threads in committers for parallel operations on files
+ * (upload, commit, abort, delete...): {@value}.
+ */
+ public static final String FS_S3A_COMMITTER_THREADS =
+ "fs.s3a.committer.threads";
+ /**
+ * Default value for {@link #FS_S3A_COMMITTER_THREADS}: {@value}.
+ */
+ public static final int DEFAULT_COMMITTER_THREADS = 8;
+
+ /**
+ * Path in the cluster filesystem for temporary data: {@value}.
+ * This is for HDFS, not the local filesystem.
+ * It is only for the summary data of each file, not the actual
+ * data being committed.
+ */
+ public static final String FS_S3A_COMMITTER_STAGING_TMP_PATH =
+ "fs.s3a.committer.staging.tmp.path";
+
+
+ /**
+ * Should the staging committers abort all pending uploads to the destination
+ * directory? Default: true.
+ *
+ * Changing this is if more than one partitioned committer is
+ * writing to the same destination tree simultaneously; otherwise
+ * the first job to complete will cancel all outstanding uploads from the
+ * others. However, it may lead to leaked outstanding uploads from failed
+ * tasks. If disabled, configure the bucket lifecycle to remove uploads
+ * after a time period, and/or set up a workflow to explicitly delete
+ * entries. Otherwise there is a risk that uncommitted uploads may run up
+ * bills.
+ */
+ public static final String FS_S3A_COMMITTER_STAGING_ABORT_PENDING_UPLOADS =
+ "fs.s3a.committer.staging.abort.pending.uploads";
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java
new file mode 100644
index 0000000..9381ef1
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitOperations.java
@@ -0,0 +1,596 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import com.amazonaws.services.s3.model.MultipartUpload;
+import com.amazonaws.services.s3.model.PartETag;
+import com.amazonaws.services.s3.model.UploadPartRequest;
+import com.amazonaws.services.s3.model.UploadPartResult;
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.S3AInstrumentation;
+import org.apache.hadoop.fs.s3a.S3AUtils;
+import org.apache.hadoop.fs.s3a.WriteOperationHelper;
+import org.apache.hadoop.fs.s3a.commit.files.PendingSet;
+import org.apache.hadoop.fs.s3a.commit.files.SinglePendingCommit;
+import org.apache.hadoop.fs.s3a.commit.files.SuccessData;
+
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+/**
+ * The implementation of the various actions a committer needs.
+ * This doesn't implement the protocol/binding to a specific execution engine,
+ * just the operations needed to to build one.
+ *
+ * When invoking FS operations, it assumes that the underlying FS is
+ * handling retries and exception translation: it does not attempt to
+ * duplicate that work.
+ *
+ */
+public class CommitOperations {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ CommitOperations.class);
+
+ /**
+ * Destination filesystem.
+ */
+ private final S3AFileSystem fs;
+
+ /** Statistics. */
+ private final S3AInstrumentation.CommitterStatistics statistics;
+
+ /**
+ * Write operations for the destination fs.
+ */
+ private final WriteOperationHelper writeOperations;
+
+ /**
+ * Filter to find all {code .pendingset} files.
+ */
+ public static final PathFilter PENDINGSET_FILTER =
+ path -> path.toString().endsWith(CommitConstants.PENDINGSET_SUFFIX);
+
+ /**
+ * Filter to find all {code .pending} files.
+ */
+ public static final PathFilter PENDING_FILTER =
+ path -> path.toString().endsWith(CommitConstants.PENDING_SUFFIX);
+
+ /**
+ * Instantiate.
+ * @param fs FS to bind to
+ */
+ public CommitOperations(S3AFileSystem fs) {
+ Preconditions.checkArgument(fs != null, "null fs");
+ this.fs = fs;
+ statistics = fs.newCommitterStatistics();
+ writeOperations = fs.createWriteOperationHelper();
+ }
+
+ /**
+ * Convert an ordered list of strings to a list of index etag parts.
+ * @param tagIds list of tags
+ * @return same list, now in numbered tuples
+ */
+ public static List<PartETag> toPartEtags(List<String> tagIds) {
+ return IntStream.range(0, tagIds.size())
+ .mapToObj(i -> new PartETag(i + 1, tagIds.get(i)))
+ .collect(Collectors.toList());
+ }
+
+ @Override
+ public String toString() {
+ return "CommitOperations{" + fs.getUri() + '}';
+ }
+
+ /** @return statistics. */
+ protected S3AInstrumentation.CommitterStatistics getStatistics() {
+ return statistics;
+ }
+
+ /**
+ * Commit the operation, throwing an exception on any failure.
+ * @param commit commit to execute
+ * @throws IOException on a failure
+ */
+ public void commitOrFail(SinglePendingCommit commit) throws IOException {
+ commit(commit, commit.getFilename()).maybeRethrow();
+ }
+
+ /**
+ * Commit a single pending commit; exceptions are caught
+ * and converted to an outcome.
+ * @param commit entry to commit
+ * @param origin origin path/string for outcome text
+ * @return the outcome
+ */
+ public MaybeIOE commit(SinglePendingCommit commit, String origin) {
+ LOG.debug("Committing single commit {}", commit);
+ MaybeIOE outcome;
+ String destKey = "unknown destination";
+ try {
+ commit.validate();
+ destKey = commit.getDestinationKey();
+ long l = innerCommit(commit);
+ LOG.debug("Successful commit of file length {}", l);
+ outcome = MaybeIOE.NONE;
+ statistics.commitCompleted(commit.getLength());
+ } catch (IOException e) {
+ String msg = String.format("Failed to commit upload against %s: %s",
+ destKey, e);
+ LOG.warn(msg, e);
+ outcome = new MaybeIOE(e);
+ statistics.commitFailed();
+ } catch (Exception e) {
+ String msg = String.format("Failed to commit upload against %s," +
+ " described in %s: %s", destKey, origin, e);
+ LOG.warn(msg, e);
+ outcome = new MaybeIOE(new PathCommitException(origin, msg, e));
+ statistics.commitFailed();
+ }
+ return outcome;
+ }
+
+ /**
+ * Inner commit operation.
+ * @param commit entry to commit
+ * @return bytes committed.
+ * @throws IOException failure
+ */
+ private long innerCommit(SinglePendingCommit commit) throws IOException {
+ // finalize the commit
+ writeOperations.completeMPUwithRetries(
+ commit.getDestinationKey(),
+ commit.getUploadId(),
+ toPartEtags(commit.getEtags()),
+ commit.getLength(),
+ new AtomicInteger(0));
+ return commit.getLength();
+ }
+
+ /**
+ * Locate all files with the pending suffix under a directory.
+ * @param pendingDir directory
+ * @param recursive recursive listing?
+ * @return the list of all located entries
+ * @throws IOException if there is a problem listing the path.
+ */
+ public List<LocatedFileStatus> locateAllSinglePendingCommits(
+ Path pendingDir,
+ boolean recursive) throws IOException {
+ return listAndFilter(fs, pendingDir, recursive, PENDING_FILTER);
+ }
+
+ /**
+ * Load all single pending commits in the directory.
+ * All load failures are logged and then added to list of files which would
+ * not load.
+ * @param pendingDir directory containing commits
+ * @param recursive do a recursive scan?
+ * @return tuple of loaded entries and those pending files which would
+ * not load/validate.
+ * @throws IOException on a failure to list the files.
+ */
+ public Pair<PendingSet,
+ List<Pair<LocatedFileStatus, IOException>>>
+ loadSinglePendingCommits(Path pendingDir, boolean recursive)
+ throws IOException {
+
+ List<LocatedFileStatus> statusList = locateAllSinglePendingCommits(
+ pendingDir, recursive);
+ PendingSet commits = new PendingSet(
+ statusList.size());
+ List<Pair<LocatedFileStatus, IOException>> failures = new ArrayList<>(1);
+ for (LocatedFileStatus status : statusList) {
+ try {
+ commits.add(SinglePendingCommit.load(fs, status.getPath()));
+ } catch (IOException e) {
+ LOG.warn("Failed to load commit file {}", status.getPath(), e);
+ failures.add(Pair.of(status, e));
+ }
+ }
+ return Pair.of(commits, failures);
+ }
+
+ /**
+ * Convert any exception to an IOE, if needed.
+ * @param key key to use in a path exception
+ * @param ex exception
+ * @return an IOE, either the passed in value or a new one wrapping the other
+ * exception.
+ */
+ public IOException makeIOE(String key, Exception ex) {
+ return ex instanceof IOException
+ ? (IOException) ex
+ : new PathCommitException(key, ex.toString(), ex);
+ }
+
+ /**
+ * Abort the multipart commit supplied. This is the lower level operation
+ * which doesn't generate an outcome, instead raising an exception.
+ * @param commit pending commit to abort
+ * @throws FileNotFoundException if the abort ID is unknown
+ * @throws IOException on any failure
+ */
+ public void abortSingleCommit(SinglePendingCommit commit)
+ throws IOException {
+ String destKey = commit.getDestinationKey();
+ String origin = commit.getFilename() != null
+ ? (" defined in " + commit.getFilename())
+ : "";
+ String uploadId = commit.getUploadId();
+ LOG.info("Aborting commit to object {}{}", destKey, origin);
+ abortMultipartCommit(destKey, uploadId);
+ }
+
+ /**
+ * Create an {@code AbortMultipartUpload} request and POST it to S3,
+ * incrementing statistics afterwards.
+ * @param destKey destination key
+ * @param uploadId upload to cancel
+ * @throws FileNotFoundException if the abort ID is unknown
+ * @throws IOException on any failure
+ */
+ public void abortMultipartCommit(String destKey, String uploadId)
+ throws IOException {
+ try {
+ writeOperations.abortMultipartCommit(destKey, uploadId);
+ } finally {
+ statistics.commitAborted();
+ }
+ }
+
+ /**
+ * Enumerate all pending files in a dir/tree, abort.
+ * @param pendingDir directory of pending operations
+ * @param recursive recurse?
+ * @return the outcome of all the abort operations
+ * @throws IOException if there is a problem listing the path.
+ */
+ public MaybeIOE abortAllSinglePendingCommits(Path pendingDir,
+ boolean recursive)
+ throws IOException {
+ Preconditions.checkArgument(pendingDir != null, "null pendingDir");
+ LOG.debug("Aborting all pending commit filess under {}"
+ + " (recursive={}", pendingDir, recursive);
+ RemoteIterator<LocatedFileStatus> pendingFiles;
+ try {
+ pendingFiles = ls(pendingDir, recursive);
+ } catch (FileNotFoundException fnfe) {
+ LOG.info("No directory to abort {}", pendingDir);
+ return MaybeIOE.NONE;
+ }
+ MaybeIOE outcome = MaybeIOE.NONE;
+ if (!pendingFiles.hasNext()) {
+ LOG.debug("No files to abort under {}", pendingDir);
+ }
+ while (pendingFiles.hasNext()) {
+ Path pendingFile = pendingFiles.next().getPath();
+ if (pendingFile.getName().endsWith(CommitConstants.PENDING_SUFFIX)) {
+ try {
+ abortSingleCommit(SinglePendingCommit.load(fs, pendingFile));
+ } catch (FileNotFoundException e) {
+ LOG.debug("listed file already deleted: {}", pendingFile);
+ } catch (IOException | IllegalArgumentException e) {
+ if (outcome == null) {
+ outcome = new MaybeIOE(makeIOE(pendingFile.toString(), e));
+ }
+ } finally {
+ // quietly try to delete the pending file
+ S3AUtils.deleteQuietly(fs, pendingFile, false);
+ }
+ }
+ }
+ return outcome;
+ }
+
+ /**
+ * List files.
+ * @param path path
+ * @param recursive recursive listing?
+ * @return iterator
+ * @throws IOException failure
+ */
+ protected RemoteIterator<LocatedFileStatus> ls(Path path, boolean recursive)
+ throws IOException {
+ return fs.listFiles(path, recursive);
+ }
+
+ /**
+ * List all pending uploads to the destination FS under a path.
+ * @param dest destination path
+ * @return A list of the pending uploads to any directory under that path.
+ * @throws IOException IO failure
+ */
+ public List<MultipartUpload> listPendingUploadsUnderPath(Path dest)
+ throws IOException {
+ return fs.listMultipartUploads(fs.pathToKey(dest));
+ }
+
+ /**
+ * Abort all pending uploads to the destination FS under a path.
+ * @param dest destination path
+ * @return a count of the number of uploads aborted.
+ * @throws IOException IO failure
+ */
+ public int abortPendingUploadsUnderPath(Path dest) throws IOException {
+ return writeOperations.abortMultipartUploadsUnderPath(fs.pathToKey(dest));
+ }
+
+ /**
+ * Delete any existing {@code _SUCCESS} file.
+ * @param outputPath output directory
+ * @throws IOException IO problem
+ */
+ public void deleteSuccessMarker(Path outputPath) throws IOException {
+ fs.delete(new Path(outputPath, _SUCCESS), false);
+ }
+
+ /**
+ * Save the success data to the {@code _SUCCESS} file.
+ * @param outputPath output directory
+ * @param successData success data to save.
+ * @param addMetrics should the FS metrics be added?
+ * @throws IOException IO problem
+ */
+ public void createSuccessMarker(Path outputPath,
+ SuccessData successData,
+ boolean addMetrics)
+ throws IOException {
+ Preconditions.checkArgument(outputPath != null, "null outputPath");
+
+ if (addMetrics) {
+ addFileSystemStatistics(successData.getMetrics());
+ }
+ // add any diagnostics
+ Configuration conf = fs.getConf();
+ successData.addDiagnostic(S3_METADATA_STORE_IMPL,
+ conf.getTrimmed(S3_METADATA_STORE_IMPL, ""));
+ successData.addDiagnostic(METADATASTORE_AUTHORITATIVE,
+ conf.getTrimmed(METADATASTORE_AUTHORITATIVE, "false"));
+ successData.addDiagnostic(MAGIC_COMMITTER_ENABLED,
+ conf.getTrimmed(MAGIC_COMMITTER_ENABLED, "false"));
+
+ // now write
+ Path markerPath = new Path(outputPath, _SUCCESS);
+ LOG.debug("Touching success marker for job {}: {}", markerPath,
+ successData);
+ successData.save(fs, markerPath, true);
+ }
+
+ /**
+ * Revert a pending commit by deleting the destination.
+ * @param commit pending commit
+ * @throws IOException failure
+ */
+ public void revertCommit(SinglePendingCommit commit) throws IOException {
+ LOG.warn("Revert {}", commit);
+ try {
+ writeOperations.revertCommit(commit.getDestinationKey());
+ } finally {
+ statistics.commitReverted();
+ }
+ }
+
+ /**
+ * Upload all the data in the local file, returning the information
+ * needed to commit the work.
+ * @param localFile local file (be a file)
+ * @param destPath destination path
+ * @param partition partition/subdir. Not used
+ * @param uploadPartSize size of upload
+ * @return a pending upload entry
+ * @throws IOException failure
+ */
+ public SinglePendingCommit uploadFileToPendingCommit(File localFile,
+ Path destPath,
+ String partition,
+ long uploadPartSize)
+ throws IOException {
+
+ LOG.debug("Initiating multipart upload from {} to {}",
+ localFile, destPath);
+ Preconditions.checkArgument(destPath != null);
+ if (!localFile.isFile()) {
+ throw new FileNotFoundException("Not a file: " + localFile);
+ }
+ String destURI = destPath.toString();
+ String destKey = fs.pathToKey(destPath);
+ String uploadId = null;
+
+ boolean threw = true;
+ try {
+ statistics.commitCreated();
+ uploadId = writeOperations.initiateMultiPartUpload(destKey);
+ long length = localFile.length();
+
+ SinglePendingCommit commitData = new SinglePendingCommit();
+ commitData.setDestinationKey(destKey);
+ commitData.setBucket(fs.getBucket());
+ commitData.touch(System.currentTimeMillis());
+ commitData.setUploadId(uploadId);
+ commitData.setUri(destURI);
+ commitData.setText(partition != null ? "partition: " + partition : "");
+ commitData.setLength(length);
+
+ long offset = 0;
+ long numParts = (length / uploadPartSize +
+ ((length % uploadPartSize) > 0 ? 1 : 0));
+ // always write one part, even if it is just an empty one
+ if (numParts == 0) {
+ numParts = 1;
+ }
+
+ List<PartETag> parts = new ArrayList<>((int) numParts);
+
+ LOG.debug("File size is {}, number of parts to upload = {}",
+ length, numParts);
+ for (int partNumber = 1; partNumber <= numParts; partNumber += 1) {
+ long size = Math.min(length - offset, uploadPartSize);
+ UploadPartRequest part;
+ part = writeOperations.newUploadPartRequest(
+ destKey,
+ uploadId,
+ partNumber,
+ (int) size,
+ null,
+ localFile,
+ offset);
+ part.setLastPart(partNumber == numParts);
+ UploadPartResult partResult = writeOperations.uploadPart(part);
+ offset += uploadPartSize;
+ parts.add(partResult.getPartETag());
+ }
+
+ commitData.bindCommitData(parts);
+ statistics.commitUploaded(length);
+ threw = false;
+ return commitData;
+ } finally {
+ if (threw && uploadId != null) {
+ statistics.commitAborted();
+ try {
+ abortMultipartCommit(destKey, uploadId);
+ } catch (IOException e) {
+ LOG.error("Failed to abort upload {} to {}", uploadId, destKey, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Add the filesystem statistics to the map; overwriting anything
+ * with the same name.
+ * @param dest destination map
+ */
+ public void addFileSystemStatistics(Map<String, Long> dest) {
+ dest.putAll(fs.getInstrumentation().toMap());
+ }
+
+ /**
+ * Note that a task has completed.
+ * @param success success flag
+ */
+ public void taskCompleted(boolean success) {
+ statistics.taskCompleted(success);
+ }
+
+ /**
+ * Note that a job has completed.
+ * @param success success flag
+ */
+ public void jobCompleted(boolean success) {
+ statistics.jobCompleted(success);
+ }
+
+ /**
+ * A holder for a possible IOException; the call {@link #maybeRethrow()}
+ * will throw any exception passed into the constructor, and be a no-op
+ * if none was.
+ *
+ * Why isn't a Java 8 optional used here? The main benefit would be that
+ * {@link #maybeRethrow()} could be done as a map(), but because Java doesn't
+ * allow checked exceptions in a map, the following code is invalid
+ * <pre>
+ * exception.map((e) -> {throw e;}
+ * </pre>
+ * As a result, the code to work with exceptions would be almost as convoluted
+ * as the original.
+ */
+ public static class MaybeIOE {
+ private final IOException exception;
+
+ public static final MaybeIOE NONE = new MaybeIOE(null);
+
+ /**
+ * Construct with an exception.
+ * @param exception exception
+ */
+ public MaybeIOE(IOException exception) {
+ this.exception = exception;
+ }
+
+ /**
+ * Get any exception.
+ * @return the exception.
+ */
+ public IOException getException() {
+ return exception;
+ }
+
+ /**
+ * Is there an exception in this class?
+ * @return true if there is an exception
+ */
+ public boolean hasException() {
+ return exception != null;
+ }
+
+ /**
+ * Rethrow any exception.
+ * @throws IOException the exception field, if non-null.
+ */
+ public void maybeRethrow() throws IOException {
+ if (exception != null) {
+ throw exception;
+ }
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder("MaybeIOE{");
+ sb.append(hasException() ? exception : "");
+ sb.append('}');
+ return sb.toString();
+ }
+
+ /**
+ * Get an instance based on the exception: either a value
+ * or a reference to {@link #NONE}.
+ * @param ex exception
+ * @return an instance.
+ */
+ public static MaybeIOE of(IOException ex) {
+ return ex != null ? new MaybeIOE(ex) : NONE;
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtils.java
new file mode 100644
index 0000000..9c684c7
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtils.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.*;
+import static org.apache.hadoop.fs.s3a.commit.ValidationFailure.verify;
+
+/**
+ * Static utility methods related to S3A commitment processing, both
+ * staging and magic.
+ *
+ * <b>Do not use in any codepath intended to be used from the S3AFS
+ * except in the committers themselves.</b>
+ */
+public final class CommitUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CommitUtils.class);
+
+ private CommitUtils() {
+ }
+
+ /**
+ * Verify that the path is a magic one.
+ * @param fs filesystem
+ * @param path path
+ * @throws PathCommitException if the path isn't a magic commit path
+ */
+ public static void verifyIsMagicCommitPath(S3AFileSystem fs,
+ Path path) throws PathCommitException {
+ verifyIsMagicCommitFS(fs);
+ if (!fs.isMagicCommitPath(path)) {
+ throw new PathCommitException(path, E_BAD_PATH);
+ }
+ }
+
+ /**
+ * Verify that an S3A FS instance is a magic commit FS.
+ * @param fs filesystem
+ * @throws PathCommitException if the FS isn't a magic commit FS.
+ */
+ public static void verifyIsMagicCommitFS(S3AFileSystem fs)
+ throws PathCommitException {
+ if (!fs.isMagicCommitEnabled()) {
+ // dump out details to console for support diagnostics
+ String fsUri = fs.getUri().toString();
+ LOG.error("{}: {}:\n{}", E_NORMAL_FS, fsUri, fs);
+ // then fail
+ throw new PathCommitException(fsUri, E_NORMAL_FS);
+ }
+ }
+
+ /**
+ * Verify that an FS is an S3A FS.
+ * @param fs filesystem
+ * @param path path to to use in exception
+ * @return the typecast FS.
+ * @throws PathCommitException if the FS is not an S3A FS.
+ */
+ public static S3AFileSystem verifyIsS3AFS(FileSystem fs, Path path)
+ throws PathCommitException {
+ if (!(fs instanceof S3AFileSystem)) {
+ throw new PathCommitException(path, E_WRONG_FS);
+ }
+ return (S3AFileSystem) fs;
+ }
+
+ /**
+ * Get the S3A FS of a path.
+ * @param path path to examine
+ * @param conf config
+ * @param magicCommitRequired is magic complete required in the FS?
+ * @return the filesystem
+ * @throws PathCommitException output path isn't to an S3A FS instance, or
+ * if {@code magicCommitRequired} is set, if doesn't support these commits.
+ * @throws IOException failure to instantiate the FS
+ */
+ public static S3AFileSystem getS3AFileSystem(Path path,
+ Configuration conf,
+ boolean magicCommitRequired)
+ throws PathCommitException, IOException {
+ S3AFileSystem s3AFS = verifyIsS3AFS(path.getFileSystem(conf), path);
+ if (magicCommitRequired) {
+ verifyIsMagicCommitFS(s3AFS);
+ }
+ return s3AFS;
+ }
+
+ /**
+ * Verify that all instances in a collection are of the given class.
+ * @param it iterator
+ * @param classname classname to require
+ * @throws ValidationFailure on a failure
+ */
+ public static void validateCollectionClass(Iterable it, Class classname)
+ throws ValidationFailure {
+ for (Object o : it) {
+ verify(o.getClass().equals(classname),
+ "Collection element is not a %s: %s", classname, o.getClass());
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java
new file mode 100644
index 0000000..c6c0da8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/CommitUtilsWithMR.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.TEMP_DATA;
+
+/**
+ * These are commit utility methods which import classes from
+ * hadoop-mapreduce, and so only work when that module is on the
+ * classpath.
+ *
+ * <b>Do not use in any codepath intended to be used from the S3AFS
+ * except in the committers themselves.</b>
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class CommitUtilsWithMR {
+
+ private CommitUtilsWithMR() {
+ }
+
+ /**
+ * Get the location of magic job attempts.
+ * @param out the base output directory.
+ * @return the location of magic job attempts.
+ */
+ public static Path getMagicJobAttemptsPath(Path out) {
+ return new Path(out, MAGIC);
+ }
+
+ /**
+ * Get the Application Attempt ID for this job.
+ * @param context the context to look in
+ * @return the Application Attempt ID for a given job.
+ */
+ public static int getAppAttemptId(JobContext context) {
+ return context.getConfiguration().getInt(
+ MRJobConfig.APPLICATION_ATTEMPT_ID, 0);
+ }
+
+ /**
+ * Compute the "magic" path for a job attempt.
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @param dest the final output directory
+ * @return the path to store job attempt data.
+ */
+ public static Path getMagicJobAttemptPath(int appAttemptId, Path dest) {
+ return new Path(getMagicJobAttemptsPath(dest),
+ formatAppAttemptDir(appAttemptId));
+ }
+
+ /**
+ * Format the application attempt directory.
+ * @param attemptId attempt ID
+ * @return the directory name for the application attempt
+ */
+ public static String formatAppAttemptDir(int attemptId) {
+ return String.format("app-attempt-%04d", attemptId);
+ }
+
+ /**
+ * Compute the path where the output of magic task attempts are stored.
+ * @param context the context of the job with magic tasks.
+ * @param dest destination of work
+ * @return the path where the output of magic task attempts are stored.
+ */
+ public static Path getMagicTaskAttemptsPath(JobContext context, Path dest) {
+ return new Path(getMagicJobAttemptPath(
+ getAppAttemptId(context), dest), "tasks");
+ }
+
+ /**
+ * Compute the path where the output of a task attempt is stored until
+ * that task is committed.
+ * This path is marked as a base path for relocations, so subdirectory
+ * information is preserved.
+ * @param context the context of the task attempt.
+ * @param dest The output path to commit work into
+ * @return the path where a task attempt should be stored.
+ */
+ public static Path getMagicTaskAttemptPath(TaskAttemptContext context,
+ Path dest) {
+ return new Path(getBaseMagicTaskAttemptPath(context, dest), BASE);
+ }
+
+ /**
+ * Get the base Magic attempt path, without any annotations to mark relative
+ * references.
+ * @param context task context.
+ * @param dest The output path to commit work into
+ * @return the path under which all attempts go
+ */
+ public static Path getBaseMagicTaskAttemptPath(TaskAttemptContext context,
+ Path dest) {
+ return new Path(getMagicTaskAttemptsPath(context, dest),
+ String.valueOf(context.getTaskAttemptID()));
+ }
+
+ /**
+ * Compute a path for temporary data associated with a job.
+ * This data is <i>not magic</i>
+ * @param appAttemptId the ID of the application attempt for this job.
+ * @param out output directory of job
+ * @return the path to store temporary job attempt data.
+ */
+ public static Path getTempJobAttemptPath(int appAttemptId, Path out) {
+ return new Path(new Path(out, TEMP_DATA),
+ formatAppAttemptDir(appAttemptId));
+ }
+
+ /**
+ * Compute the path where the output of a given job attempt will be placed.
+ * @param context task context
+ * @param out output directory of job
+ * @return the path to store temporary job attempt data.
+ */
+ public static Path getTempTaskAttemptPath(TaskAttemptContext context,
+ Path out) {
+ return new Path(getTempJobAttemptPath(getAppAttemptId(context), out),
+ String.valueOf(context.getTaskAttemptID()));
+ }
+
+ /**
+ * Get a string value of a job ID; returns meaningful text if there is no ID.
+ * @param context job context
+ * @return a string for logs
+ */
+ public static String jobIdString(JobContext context) {
+ JobID jobID = context.getJobID();
+ return jobID != null ? jobID.toString() : "(no job ID)";
+ }
+
+ /**
+ * Get a job name; returns meaningful text if there is no name.
+ * @param context job context
+ * @return a string for logs
+ */
+ public static String jobName(JobContext context) {
+ String name = context.getJobName();
+ return (name != null && !name.isEmpty()) ? name : "(anonymous)";
+ }
+
+ /**
+ * Get a configuration option, with any value in the job configuration
+ * taking priority over that in the filesystem.
+ * This allows for per-job override of FS parameters.
+ *
+ * Order is: job context, filesystem config, default value
+ *
+ * @param context job/task context
+ * @param fsConf filesystem configuration. Get this from the FS to guarantee
+ * per-bucket parameter propagation
+ * @param key key to look for
+ * @param defVal default value
+ * @return the configuration option.
+ */
+ public static String getConfigurationOption(
+ JobContext context,
+ Configuration fsConf,
+ String key,
+ String defVal) {
+ return context.getConfiguration().getTrimmed(key,
+ fsConf.getTrimmed(key, defVal));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Duration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Duration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Duration.java
new file mode 100644
index 0000000..c44a90b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/Duration.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+/**
+ * Little duration counter.
+ */
+public class Duration {
+
+ private final long started;
+ private long finished;
+
+ public Duration() {
+ started = time();
+ finished = started;
+ }
+
+ protected long time() {
+ return System.currentTimeMillis();
+ }
+
+ public void finished() {
+ finished = time();
+ }
+
+ public String getDurationString() {
+ return humanTime(value());
+ }
+
+ public static String humanTime(long time) {
+ long seconds = (time / 1000);
+ long minutes = (seconds / 60);
+ return String.format("%d:%02d.%03ds", minutes, seconds % 60, time % 1000);
+ }
+
+ @Override
+ public String toString() {
+ return getDurationString();
+ }
+
+ public long value() {
+ return finished -started;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java
new file mode 100644
index 0000000..c6617f8
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/DurationInfo.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.slf4j.Logger;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * A duration with logging of final state at info in the {@code close()} call.
+ * This allows it to be used in a try-with-resources clause, and have the
+ * duration automatically logged.
+ */
+@InterfaceAudience.Private
+public class DurationInfo extends Duration
+ implements AutoCloseable {
+ private final String text;
+
+ private final Logger log;
+
+ /**
+ * Create the duration text from a {@code String.format()} code call.
+ * @param log log to write to
+ * @param format format string
+ * @param args list of arguments
+ */
+ public DurationInfo(Logger log, String format, Object... args) {
+ this.text = String.format(format, args);
+ this.log = log;
+ log.info("Starting: {}", text);
+ }
+
+ @Override
+ public String toString() {
+ return text + ": duration " + super.toString();
+ }
+
+ @Override
+ public void close() {
+ finished();
+ log.info(this.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java
new file mode 100644
index 0000000..2821fce
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/InternalCommitterConstants.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitterFactory;
+import org.apache.hadoop.fs.s3a.commit.staging.DirectoryStagingCommitterFactory;
+import org.apache.hadoop.fs.s3a.commit.staging.PartitionedStagingCommitterFactory;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterFactory;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC_COMMITTER_ENABLED;
+
+/**
+ * These are internal constants not intended for public use.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public final class InternalCommitterConstants {
+
+ private InternalCommitterConstants() {
+ }
+
+ /**
+ * This is the staging committer base class; only used for testing.
+ */
+ public static final String COMMITTER_NAME_STAGING = "staging";
+
+ /**
+ * A unique identifier to use for this work: {@value}.
+ */
+ public static final String FS_S3A_COMMITTER_STAGING_UUID =
+ "fs.s3a.committer.staging.uuid";
+
+ /**
+ * Directory committer factory: {@value}.
+ */
+ public static final String STAGING_COMMITTER_FACTORY =
+ StagingCommitterFactory.CLASSNAME;
+
+ /**
+ * Directory committer factory: {@value}.
+ */
+ public static final String DIRECTORY_COMMITTER_FACTORY =
+ DirectoryStagingCommitterFactory.CLASSNAME;
+
+ /**
+ * Partitioned committer factory: {@value}.
+ */
+ public static final String PARTITION_COMMITTER_FACTORY =
+ PartitionedStagingCommitterFactory.CLASSNAME;
+
+ /**
+ * Magic committer factory: {@value}.
+ */
+ public static final String MAGIC_COMMITTER_FACTORY =
+ MagicS3GuardCommitterFactory.CLASSNAME;
+
+ /**
+ * Error text when the destination path exists and the committer
+ * must abort the job/task {@value}.
+ */
+ public static final String E_DEST_EXISTS =
+ "Destination path exists and committer conflict resolution mode is "
+ + "\"fail\"";
+
+ /** Error message for bad path: {@value}. */
+ public static final String E_BAD_PATH
+ = "Path does not represent a magic-commit path";
+
+ /** Error message if filesystem isn't magic: {@value}. */
+ public static final String E_NORMAL_FS
+ = "Filesystem does not have support for 'magic' committer enabled"
+ + " in configuration option " + MAGIC_COMMITTER_ENABLED;
+
+ /** Error message if the dest FS isn't S3A: {@value}. */
+ public static final String E_WRONG_FS
+ = "Output path is not on an S3A Filesystem";
+
+ /** Error message for a path without a magic element in the list: {@value}. */
+ public static final String E_NO_MAGIC_PATH_ELEMENT
+ = "No " + MAGIC + " element in path";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/LocalTempDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/LocalTempDir.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/LocalTempDir.java
new file mode 100644
index 0000000..849a06d
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/LocalTempDir.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.Constants;
+
+import static org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR;
+
+/**
+ * A class which manages access to a temporary directory store, uses the
+ * directories listed in {@link Constants#BUFFER_DIR} for this.
+ */
+final class LocalTempDir {
+
+ private LocalTempDir() {
+ }
+
+ private static LocalDirAllocator directoryAllocator;
+
+ private static synchronized LocalDirAllocator getAllocator(
+ Configuration conf, String key) {
+ if (directoryAllocator != null) {
+ String bufferDir = conf.get(key) != null
+ ? key : Constants.HADOOP_TMP_DIR;
+ directoryAllocator = new LocalDirAllocator(bufferDir);
+ }
+ return directoryAllocator;
+ }
+
+ /**
+ * Create a temp file.
+ * @param conf configuration to use when creating the allocator
+ * @param prefix filename prefix
+ * @param size file size, or -1 if not known
+ * @return the temp file. The file has been created.
+ * @throws IOException IO failure
+ */
+ public static File tempFile(Configuration conf, String prefix, long size)
+ throws IOException {
+ return getAllocator(conf, BUFFER_DIR).createTmpFileForWrite(
+ prefix, size, conf);
+ }
+
+ /**
+ * Get a temporary path.
+ * @param conf configuration to use when creating the allocator
+ * @param prefix filename prefix
+ * @param size file size, or -1 if not known
+ * @return the temp path.
+ * @throws IOException IO failure
+ */
+ public static Path tempPath(Configuration conf, String prefix, long size)
+ throws IOException {
+ return getAllocator(conf, BUFFER_DIR)
+ .getLocalPathForWrite(prefix, size, conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java
new file mode 100644
index 0000000..a07b5c9
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitIntegration.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.Statistic;
+import org.apache.hadoop.fs.s3a.commit.magic.MagicCommitTracker;
+
+import static org.apache.hadoop.fs.s3a.commit.MagicCommitPaths.*;
+
+/**
+ * Adds the code needed for S3A to support magic committers.
+ * It's pulled out to keep S3A FS class slightly less complex.
+ * This class can be instantiated even when magic commit is disabled;
+ * in this case:
+ * <ol>
+ * <li>{@link #isMagicCommitPath(Path)} will always return false.</li>
+ * <li>{@link #createTracker(Path, String)} will always return an instance
+ * of {@link PutTracker}.</li>
+ * </ol>
+ *
+ * <p>Important</p>: must not directly or indirectly import a class which
+ * uses any datatype in hadoop-mapreduce.
+ */
+public class MagicCommitIntegration {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(MagicCommitIntegration.class);
+ private final S3AFileSystem owner;
+ private final boolean magicCommitEnabled;
+
+ /**
+ * Instantiate.
+ * @param owner owner class
+ * @param magicCommitEnabled is magic commit enabled.
+ */
+ public MagicCommitIntegration(S3AFileSystem owner,
+ boolean magicCommitEnabled) {
+ this.owner = owner;
+ this.magicCommitEnabled = magicCommitEnabled;
+ }
+
+ /**
+ * Given an (elements, key) pair, return the key of the final destination of
+ * the PUT, that is: where the final path is expected to go?
+ * @param elements path split to elements
+ * @param key key
+ * @return key for final put. If this is not a magic commit, the
+ * same as the key in.
+ */
+ public String keyOfFinalDestination(List<String> elements, String key) {
+ if (isMagicCommitPath(elements)) {
+ return elementsToKey(finalDestination(elements));
+ } else {
+ return key;
+ }
+ }
+
+ /**
+ * Given a path and a key to that same path, create a tracker for it.
+ * This specific tracker will be chosen based on whether or not
+ * the path is a magic one.
+ * @param path path of nominal write
+ * @param key key of path of nominal write
+ * @return the tracker for this operation.
+ */
+ public PutTracker createTracker(Path path, String key) {
+ final List<String> elements = splitPathToElements(path);
+ PutTracker tracker;
+
+ if(isMagicFile(elements)) {
+ // path is of a magic file
+ if (isMagicCommitPath(elements)) {
+ final String destKey = keyOfFinalDestination(elements, key);
+ String pendingsetPath = key + CommitConstants.PENDING_SUFFIX;
+ owner.getInstrumentation()
+ .incrementCounter(Statistic.COMMITTER_MAGIC_FILES_CREATED, 1);
+ tracker = new MagicCommitTracker(path,
+ owner.getBucket(),
+ key,
+ destKey,
+ pendingsetPath,
+ owner.createWriteOperationHelper());
+ LOG.debug("Created {}", tracker);
+ } else {
+ LOG.warn("File being created has a \"magic\" path, but the filesystem"
+ + " has magic file support disabled: {}", path);
+ // downgrade to standard multipart tracking
+ tracker = new PutTracker(key);
+ }
+ } else {
+ // standard multipart tracking
+ tracker = new PutTracker(key);
+ }
+ return tracker;
+ }
+
+ /**
+ * This performs the calculation of the final destination of a set
+ * of elements.
+ *
+ * @param elements original (do not edit after this call)
+ * @return a list of elements, possibly empty
+ */
+ private List<String> finalDestination(List<String> elements) {
+ return magicCommitEnabled ?
+ MagicCommitPaths.finalDestination(elements)
+ : elements;
+ }
+
+ /**
+ * Is magic commit enabled?
+ * @return true if magic commit is turned on.
+ */
+ public boolean isMagicCommitEnabled() {
+ return magicCommitEnabled;
+ }
+
+ /**
+ * Predicate: is a path a magic commit path?
+ * @param path path to examine
+ * @return true if the path is or is under a magic directory
+ */
+ public boolean isMagicCommitPath(Path path) {
+ return isMagicCommitPath(splitPathToElements(path));
+ }
+
+ /**
+ * Is this path a magic commit path in this filesystem?
+ * True if magic commit is enabled, the path is magic
+ * and the path is not actually a commit metadata file.
+ * @param elements element list
+ * @return true if writing path is to be uprated to a magic file write
+ */
+ private boolean isMagicCommitPath(List<String> elements) {
+ return magicCommitEnabled && isMagicFile(elements);
+ }
+
+ /**
+ * Is the file a magic file: this predicate doesn't check
+ * for the FS actually having the magic bit being set.
+ * @param elements path elements
+ * @return true if the path is one a magic file write expects.
+ */
+ private boolean isMagicFile(List<String> elements) {
+ return isMagicPath(elements) &&
+ !isCommitMetadataFile(elements);
+ }
+
+ /**
+ * Does this file contain all the commit metadata?
+ * @param elements path element list
+ * @return true if this file is one of the commit metadata files.
+ */
+ private boolean isCommitMetadataFile(List<String> elements) {
+ String last = elements.get(elements.size() - 1);
+ return last.endsWith(CommitConstants.PENDING_SUFFIX)
+ || last.endsWith(CommitConstants.PENDINGSET_SUFFIX);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java
new file mode 100644
index 0000000..745b5b2
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/MagicCommitPaths.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.BASE;
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.MAGIC;
+import static org.apache.hadoop.fs.s3a.commit.InternalCommitterConstants.E_NO_MAGIC_PATH_ELEMENT;
+
+/**
+ * Operations on (magic) paths.
+ */
+public final class MagicCommitPaths {
+
+ private MagicCommitPaths() {
+ }
+
+ /**
+ * Take an absolute path, split it into a list of elements.
+ * If empty, the path is the root path.
+ * @param path input path
+ * @return a possibly empty list of elements.
+ * @throws IllegalArgumentException if the path is invalid -relative, empty...
+ */
+ public static List<String> splitPathToElements(Path path) {
+ checkArgument(path.isAbsolute(), "path is relative");
+ String uriPath = path.toUri().getPath();
+ checkArgument(!uriPath.isEmpty(), "empty path");
+ if ("/".equals(uriPath)) {
+ // special case: empty list
+ return new ArrayList<>(0);
+ }
+ List<String> elements = new ArrayList<>();
+ int len = uriPath.length();
+ int firstElementChar = 1;
+ int endOfElement = uriPath.indexOf('/', firstElementChar);
+ while (endOfElement > 0) {
+ elements.add(uriPath.substring(firstElementChar, endOfElement));
+ firstElementChar = endOfElement + 1;
+ endOfElement = firstElementChar == len ? -1
+ : uriPath.indexOf('/', firstElementChar);
+ }
+ // expect a possible child element here
+ if (firstElementChar != len) {
+ elements.add(uriPath.substring(firstElementChar));
+ }
+ return elements;
+ }
+
+ /**
+ * Is a path in the magic tree?
+ * @param elements element list
+ * @return true if a path is considered magic
+ */
+ public static boolean isMagicPath(List<String> elements) {
+ return elements.contains(MAGIC);
+ }
+
+ /**
+ * Does the list of magic elements contain a base path marker?
+ * @param elements element list, already stripped out
+ * from the magic tree.
+ * @return true if a path has a base directory
+ */
+ public static boolean containsBasePath(List<String> elements) {
+ return elements.contains(BASE);
+ }
+
+ /**
+ * Get the index of the magic path element.
+ * @param elements full path element list
+ * @return the index.
+ * @throws IllegalArgumentException if there is no magic element
+ */
+ public static int magicElementIndex(List<String> elements) {
+ int index = elements.indexOf(MAGIC);
+ checkArgument(index >= 0, E_NO_MAGIC_PATH_ELEMENT);
+ return index;
+ }
+
+ /**
+ * Get the parent path elements of the magic path.
+ * The list may be immutable or may be a view of the underlying list.
+ * Both the parameter list and the returned list MUST NOT be modified.
+ * @param elements full path element list
+ * @return the parent elements; may be empty
+ */
+ public static List<String> magicPathParents(List<String> elements) {
+ return elements.subList(0, magicElementIndex(elements));
+ }
+
+ /**
+ * Get the child path elements under the magic path.
+ * The list may be immutable or may be a view of the underlying list.
+ * Both the parameter list and the returned list MUST NOT be modified.
+ * @param elements full path element list
+ * @return the child elements; may be empty
+ */
+ public static List<String> magicPathChildren(List<String> elements) {
+ int index = magicElementIndex(elements);
+ int len = elements.size();
+ if (index == len - 1) {
+ // empty index
+ return Collections.emptyList();
+ } else {
+ return elements.subList(index + 1, len);
+ }
+ }
+
+ /**
+ * Get any child path elements under any {@code __base} path,
+ * or an empty list if there is either: no {@code __base} path element,
+ * or no child entries under it.
+ * The list may be immutable or may be a view of the underlying list.
+ * Both the parameter list and the returned list MUST NOT be modified.
+ * @param elements full path element list
+ * @return the child elements; may be empty
+ */
+ public static List<String> basePathChildren(List<String> elements) {
+ int index = elements.indexOf(BASE);
+ if (index < 0) {
+ return Collections.emptyList();
+ }
+ int len = elements.size();
+ if (index == len - 1) {
+ // empty index
+ return Collections.emptyList();
+ } else {
+ return elements.subList(index + 1, len);
+ }
+ }
+
+ /**
+ * Take a list of elements and create an S3 key by joining them
+ * with "/" between each one.
+ * @param elements path elements
+ * @return a path which can be used in the AWS API
+ */
+ public static String elementsToKey(List<String> elements) {
+ return StringUtils.join("/", elements);
+ }
+
+ /**
+ * Get the filename of a path: the last element.
+ * @param elements element list.
+ * @return the filename; the last element.
+ */
+ public static String filename(List<String> elements) {
+ return lastElement(elements);
+ }
+
+ /**
+ * Last element of a (non-empty) list.
+ * @param strings strings in
+ * @return the last one.
+ */
+ public static String lastElement(List<String> strings) {
+ checkArgument(!strings.isEmpty(), "empty list");
+ return strings.get(strings.size() - 1);
+ }
+
+ /**
+ * Get the magic subdirectory of a destination directory.
+ * @param destDir the destination directory
+ * @return a new path.
+ */
+ public static Path magicSubdir(Path destDir) {
+ return new Path(destDir, MAGIC);
+ }
+
+ /**
+ * Calculates the final destination of a file.
+ * This is the parent of any {@code __magic} element, and the filename
+ * of the path. That is: all intermediate child path elements are discarded.
+ * Why so? paths under the magic path include job attempt and task attempt
+ * subdirectories, which need to be skipped.
+ *
+ * If there is a {@code __base} directory in the children, then it becomes
+ * a base for unflattened paths, that is: all its children are pulled into
+ * the final destination.
+ * @param elements element list.
+ * @return the path
+ */
+ public static List<String> finalDestination(List<String> elements) {
+ if (isMagicPath(elements)) {
+ List<String> destDir = magicPathParents(elements);
+ List<String> children = magicPathChildren(elements);
+ checkArgument(!children.isEmpty(), "No path found under " +
+ MAGIC);
+ ArrayList<String> dest = new ArrayList<>(destDir);
+ if (containsBasePath(children)) {
+ // there's a base marker in the path
+ List<String> baseChildren = basePathChildren(children);
+ checkArgument(!baseChildren.isEmpty(),
+ "No path found under " + BASE);
+ dest.addAll(baseChildren);
+ } else {
+ dest.add(filename(children));
+ }
+ return dest;
+ } else {
+ return elements;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PathCommitException.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PathCommitException.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PathCommitException.java
new file mode 100644
index 0000000..4607548
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PathCommitException.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathIOException;
+
+/**
+ * Path exception to use for various commit issues.
+ */
+public class PathCommitException extends PathIOException {
+ public PathCommitException(String path, Throwable cause) {
+ super(path, cause);
+ }
+
+ public PathCommitException(String path, String error) {
+ super(path, error);
+ }
+
+ public PathCommitException(Path path, String error) {
+ super(path != null ? path.toString() : "", error);
+ }
+
+ public PathCommitException(String path, String error, Throwable cause) {
+ super(path, error, cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java
new file mode 100644
index 0000000..bbffef3
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/PutTracker.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+import java.util.List;
+
+import com.amazonaws.services.s3.model.PartETag;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * Multipart put tracker.
+ * Base class does nothing except declare that any
+ * MPU must complete in the {@code close()} operation.
+ *
+ */
+@InterfaceAudience.Private
+public class PutTracker {
+
+ /** The destination. */
+ private final String destKey;
+
+ /**
+ * Instantiate.
+ * @param destKey destination key
+ */
+ public PutTracker(String destKey) {
+ this.destKey = destKey;
+ }
+
+ /**
+ * Startup event.
+ * @return true if the multipart should start immediately.
+ * @throws IOException any IO problem.
+ */
+ public boolean initialize() throws IOException {
+ return false;
+ }
+
+ /**
+ * Flag to indicate that output is not immediately visible after the stream
+ * is closed. Default: false.
+ * @return true if the output's visibility will be delayed.
+ */
+ public boolean outputImmediatelyVisible() {
+ return true;
+ }
+
+ /**
+ * Callback when the upload is is about to complete.
+ * @param uploadId Upload ID
+ * @param parts list of parts
+ * @param bytesWritten bytes written
+ * @return true if the commit is to be initiated immediately.
+ * False implies the output stream does not need to worry about
+ * what happens.
+ * @throws IOException I/O problem or validation failure.
+ */
+ public boolean aboutToComplete(String uploadId,
+ List<PartETag> parts,
+ long bytesWritten)
+ throws IOException {
+ return true;
+ }
+
+ /**
+ * get the destination key. The default implementation returns the
+ * key passed in: there is no adjustment of the destination.
+ * @return the destination to use in PUT requests.
+ */
+ public String getDestKey() {
+ return destKey;
+ }
+
+ @Override
+ public String toString() {
+ final StringBuilder sb = new StringBuilder(
+ "DefaultPutTracker{");
+ sb.append("destKey='").append(destKey).append('\'');
+ sb.append('}');
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de8b6ca5/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/S3ACommitterFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/S3ACommitterFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/S3ACommitterFactory.java
new file mode 100644
index 0000000..6b170f9
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/S3ACommitterFactory.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a.commit;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.s3a.S3AFileSystem;
+import org.apache.hadoop.fs.s3a.commit.magic.MagicS3GuardCommitterFactory;
+import org.apache.hadoop.fs.s3a.commit.staging.DirectoryStagingCommitterFactory;
+import org.apache.hadoop.fs.s3a.commit.staging.PartitionedStagingCommitterFactory;
+import org.apache.hadoop.fs.s3a.commit.staging.StagingCommitterFactory;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.output.PathOutputCommitter;
+
+import static org.apache.hadoop.fs.s3a.commit.CommitConstants.*;
+
+/**
+ * The S3A committer factory which chooses the committer based on the
+ * specific option chosen in a per-bucket basis from the property
+ * {@link CommitConstants#FS_S3A_COMMITTER_NAME}.
+ *
+ * This should be instantiated by using the property value {@link #CLASSNAME}
+ * as the committer for the job, then set the filesystem property
+ * {@link CommitConstants#FS_S3A_COMMITTER_NAME} to one of
+ * <ul>
+ * <li>{@link CommitConstants#COMMITTER_NAME_FILE}: File committer.</li>
+ * <li>{@link CommitConstants#COMMITTER_NAME_DIRECTORY}:
+ * Staging directory committer.</li>
+ * <li>{@link CommitConstants#COMMITTER_NAME_PARTITIONED}:
+ * Staging partitioned committer.</li>
+ * <li>{@link CommitConstants#COMMITTER_NAME_MAGIC}:
+ * the "Magic" committer</li>
+ * <li>{@link InternalCommitterConstants#COMMITTER_NAME_STAGING}:
+ * the "staging" committer, which isn't intended for use outside tests.</li>
+ * </ul>
+ * There are no checks to verify that the filesystem is compatible with
+ * the committer.
+ */
+public class S3ACommitterFactory extends AbstractS3ACommitterFactory {
+
+ /**
+ * Name of this class: {@value}.
+ */
+ public static final String CLASSNAME
+ = "org.apache.hadoop.fs.s3a.commit.S3ACommitterFactory";
+
+ /**
+ * Create a task committer.
+ * @param fileSystem destination FS.
+ * @param outputPath final output path for work
+ * @param context job context
+ * @return a committer
+ * @throws IOException instantiation failure
+ */
+ @Override
+ public PathOutputCommitter createTaskCommitter(S3AFileSystem fileSystem,
+ Path outputPath,
+ TaskAttemptContext context) throws IOException {
+ AbstractS3ACommitterFactory factory = chooseCommitterFactory(fileSystem,
+ outputPath,
+ context.getConfiguration());
+ return factory != null ?
+ factory.createTaskCommitter(fileSystem, outputPath, context)
+ : createFileOutputCommitter(outputPath, context);
+ }
+
+ /**
+ * Choose a committer from the FS and task configurations. Task Configuration
+ * takes priority, allowing execution engines to dynamically change
+ * committer on a query-by-query basis.
+ * @param fileSystem FS
+ * @param outputPath destination path
+ * @param taskConf configuration from the task
+ * @return An S3A committer if chosen, or "null" for the classic value
+ * @throws PathCommitException on a failure to identify the committer
+ */
+ private AbstractS3ACommitterFactory chooseCommitterFactory(
+ S3AFileSystem fileSystem,
+ Path outputPath,
+ Configuration taskConf) throws PathCommitException {
+ AbstractS3ACommitterFactory factory;
+
+ // the FS conf will have had its per-bucket values resolved, unlike
+ // job/task configurations.
+ Configuration fsConf = fileSystem.getConf();
+
+ String name = fsConf.getTrimmed(FS_S3A_COMMITTER_NAME, COMMITTER_NAME_FILE);
+ name = taskConf.getTrimmed(FS_S3A_COMMITTER_NAME, name);
+ switch (name) {
+ case COMMITTER_NAME_FILE:
+ factory = null;
+ break;
+ case COMMITTER_NAME_DIRECTORY:
+ factory = new DirectoryStagingCommitterFactory();
+ break;
+ case COMMITTER_NAME_PARTITIONED:
+ factory = new PartitionedStagingCommitterFactory();
+ break;
+ case COMMITTER_NAME_MAGIC:
+ factory = new MagicS3GuardCommitterFactory();
+ break;
+ case InternalCommitterConstants.COMMITTER_NAME_STAGING:
+ factory = new StagingCommitterFactory();
+ break;
+ default:
+ throw new PathCommitException(outputPath,
+ "Unknown committer: \"" + name + "\"");
+ }
+ return factory;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org