You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/11/17 18:44:10 UTC
[01/19] hadoop git commit: HADOOP-15023. ValueQueue should also
validate (int) (lowWatermark * numValues) > 0 on construction. [Forced
Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-6592 32620a129 -> b9f0e942f (forced update)
HADOOP-15023. ValueQueue should also validate (int) (lowWatermark * numValues) > 0 on construction.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1941b20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1941b20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1941b20
Branch: refs/heads/YARN-6592
Commit: b1941b200d6b4fa6a7891421c0a1e212cad3d6eb
Parents: fac72ee
Author: Xiao Chen <xi...@apache.org>
Authored: Wed Nov 15 16:43:25 2017 -0800
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Nov 15 16:44:06 2017 -0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1941b20/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
index 8411ffb..1ddd8a3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/ValueQueue.java
@@ -224,6 +224,9 @@ public class ValueQueue <E> {
Preconditions.checkArgument(numValues > 0, "\"numValues\" must be > 0");
Preconditions.checkArgument(((lowWatermark > 0)&&(lowWatermark <= 1)),
"\"lowWatermark\" must be > 0 and <= 1");
+ final int watermarkValue = (int) (numValues * lowWatermark);
+ Preconditions.checkArgument(watermarkValue > 0,
+ "(int) (\"numValues\" * \"lowWatermark\") must be > 0");
Preconditions.checkArgument(expiry > 0, "\"expiry\" must be > 0");
Preconditions.checkArgument(numFillerThreads > 0,
"\"numFillerThreads\" must be > 0");
@@ -243,8 +246,7 @@ public class ValueQueue <E> {
throws Exception {
LinkedBlockingQueue<E> keyQueue =
new LinkedBlockingQueue<E>();
- refiller.fillQueueForKey(keyName, keyQueue,
- (int)(lowWatermark * numValues));
+ refiller.fillQueueForKey(keyName, keyQueue, watermarkValue);
return keyQueue;
}
});
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/19] hadoop git commit: YARN-6595. [API] Add Placement Constraints
at the application level. (Arun Suresh via kkaranasos)
Posted by as...@apache.org.
YARN-6595. [API] Add Placement Constraints at the application level. (Arun Suresh via kkaranasos)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c24deea4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c24deea4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c24deea4
Branch: refs/heads/YARN-6592
Commit: c24deea45d8db458c8a7285e346a867b49cf11dc
Parents: ff5a320
Author: Konstantinos Karanasos <kk...@apache.org>
Authored: Mon Nov 13 15:25:24 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Nov 17 10:43:44 2017 -0800
----------------------------------------------------------------------
.../RegisterApplicationMasterRequest.java | 42 ++++-
.../yarn/api/resource/PlacementConstraint.java | 156 +++++++++++++++++++
.../src/main/proto/yarn_protos.proto | 6 +
.../src/main/proto/yarn_service_protos.proto | 1 +
.../RegisterApplicationMasterRequestPBImpl.java | 106 ++++++++++++-
.../hadoop/yarn/api/BasePBImplRecordsTest.java | 11 ++
6 files changed, 313 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24deea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
index 395e190..f2d537a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
@@ -18,11 +18,16 @@
package org.apache.hadoop.yarn.api.protocolrecords;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
import org.apache.hadoop.yarn.util.Records;
-
/**
* The request sent by the {@code ApplicationMaster} to {@code ResourceManager}
* on registration.
@@ -132,4 +137,39 @@ public abstract class RegisterApplicationMasterRequest {
@Public
@Stable
public abstract void setTrackingUrl(String trackingUrl);
+
+ /**
+ * Return all Placement Constraints specified at the Application level. The
+ * mapping is from a set of allocation tags to a
+ * <code>PlacementConstraint</code> associated with the tags, i.e., each
+ * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest} that has those
+ * tags will be placed taking into account the corresponding constraint.
+ *
+ * @return A map of Placement Constraints.
+ */
+ @Public
+ @Unstable
+ public Map<Set<String>, PlacementConstraint> getPlacementConstraints() {
+ return new HashMap<>();
+ }
+
+ /**
+ * Set Placement Constraints applicable to the
+ * {@link org.apache.hadoop.yarn.api.records.SchedulingRequest}s
+ * of this application.
+ * The mapping is from a set of allocation tags to a
+ * <code>PlacementConstraint</code> associated with the tags.
+ * For example:
+ * Map <
+ * <hb_regionserver> -> node_anti_affinity,
+ * <hb_regionserver, hb_master> -> rack_affinity,
+ * ...
+ * >
+ * @param placementConstraints Placement Constraint Mapping.
+ */
+ @Public
+ @Unstable
+ public void setPlacementConstraints(
+ Map<Set<String>, PlacementConstraint> placementConstraints) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24deea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
index f0e3982..b6e851a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -54,6 +54,26 @@ public class PlacementConstraint {
return constraintExpr;
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof PlacementConstraint)) {
+ return false;
+ }
+
+ PlacementConstraint that = (PlacementConstraint) o;
+
+ return getConstraintExpr() != null ? getConstraintExpr().equals(that
+ .getConstraintExpr()) : that.getConstraintExpr() == null;
+ }
+
+ @Override
+ public int hashCode() {
+ return getConstraintExpr() != null ? getConstraintExpr().hashCode() : 0;
+ }
+
/**
* Interface used to enable the elements of the constraint tree to be visited.
*/
@@ -174,6 +194,38 @@ public class PlacementConstraint {
}
@Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof SingleConstraint)) {
+ return false;
+ }
+
+ SingleConstraint that = (SingleConstraint) o;
+
+ if (getMinCardinality() != that.getMinCardinality()) {
+ return false;
+ }
+ if (getMaxCardinality() != that.getMaxCardinality()) {
+ return false;
+ }
+ if (!getScope().equals(that.getScope())) {
+ return false;
+ }
+ return getTargetExpressions().equals(that.getTargetExpressions());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getScope().hashCode();
+ result = 31 * result + getMinCardinality();
+ result = 31 * result + getMaxCardinality();
+ result = 31 * result + getTargetExpressions().hashCode();
+ return result;
+ }
+
+ @Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@@ -332,6 +384,34 @@ public class PlacementConstraint {
}
@Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof TargetConstraint)) {
+ return false;
+ }
+
+ TargetConstraint that = (TargetConstraint) o;
+
+ if (getOp() != that.getOp()) {
+ return false;
+ }
+ if (!getScope().equals(that.getScope())) {
+ return false;
+ }
+ return getTargetExpressions().equals(that.getTargetExpressions());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getOp().hashCode();
+ result = 31 * result + getScope().hashCode();
+ result = 31 * result + getTargetExpressions().hashCode();
+ return result;
+ }
+
+ @Override
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
@@ -388,6 +468,34 @@ public class PlacementConstraint {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ CardinalityConstraint that = (CardinalityConstraint) o;
+
+ if (minCardinality != that.minCardinality) {
+ return false;
+ }
+ if (maxCardinality != that.maxCardinality) {
+ return false;
+ }
+ return scope != null ? scope.equals(that.scope) : that.scope == null;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = scope != null ? scope.hashCode() : 0;
+ result = 31 * result + minCardinality;
+ result = 31 * result + maxCardinality;
+ return result;
+ }
}
/**
@@ -406,6 +514,25 @@ public class PlacementConstraint {
* @return the children of the composite constraint
*/
public abstract List<R> getChildren();
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ return getChildren() != null ? getChildren().equals(
+ ((CompositeConstraint)o).getChildren()) :
+ ((CompositeConstraint)o).getChildren() == null;
+ }
+
+ @Override
+ public int hashCode() {
+ return getChildren() != null ? getChildren().hashCode() : 0;
+ }
}
/**
@@ -563,5 +690,34 @@ public class PlacementConstraint {
public <T> T accept(Visitor<T> visitor) {
return visitor.visit(this);
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ TimedPlacementConstraint that = (TimedPlacementConstraint) o;
+
+ if (schedulingDelay != that.schedulingDelay) {
+ return false;
+ }
+ if (constraint != null ? !constraint.equals(that.constraint) :
+ that.constraint != null) {
+ return false;
+ }
+ return delayUnit == that.delayUnit;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = constraint != null ? constraint.hashCode() : 0;
+ result = 31 * result + (int) (schedulingDelay ^ (schedulingDelay >>> 32));
+ result = 31 * result + (delayUnit != null ? delayUnit.hashCode() : 0);
+ return result;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24deea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 09feb1d..215e3ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -636,6 +636,12 @@ message CompositePlacementConstraintProto {
repeated TimedPlacementConstraintProto timedChildConstraints = 3;
}
+// This associates a set of allocation tags to a Placement Constraint.
+message PlacementConstraintMapEntryProto {
+ repeated string allocation_tags = 1;
+ optional PlacementConstraintProto placement_constraint = 2;
+}
+
////////////////////////////////////////////////////////////////////////
////// From reservation_protocol /////////////////////////////////////
////////////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24deea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 7146b99..26bcb1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -38,6 +38,7 @@ message RegisterApplicationMasterRequestProto {
optional string host = 1;
optional int32 rpc_port = 2;
optional string tracking_url = 3;
+ repeated PlacementConstraintMapEntryProto placement_constraints = 4;
}
message RegisterApplicationMasterResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24deea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java
index 037dfd9..64bee85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterRequestPBImpl.java
@@ -21,24 +21,41 @@ package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintFromProtoConverter;
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintToProtoConverter;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterRequestProtoOrBuilder;
import com.google.protobuf.TextFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
@Private
@Unstable
-public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationMasterRequest {
- RegisterApplicationMasterRequestProto proto = RegisterApplicationMasterRequestProto.getDefaultInstance();
- RegisterApplicationMasterRequestProto.Builder builder = null;
+public class RegisterApplicationMasterRequestPBImpl
+ extends RegisterApplicationMasterRequest {
+ private RegisterApplicationMasterRequestProto proto =
+ RegisterApplicationMasterRequestProto.getDefaultInstance();
+ private RegisterApplicationMasterRequestProto.Builder builder = null;
+ private Map<Set<String>, PlacementConstraint> placementConstraints = null;
boolean viaProto = false;
public RegisterApplicationMasterRequestPBImpl() {
builder = RegisterApplicationMasterRequestProto.newBuilder();
}
- public RegisterApplicationMasterRequestPBImpl(RegisterApplicationMasterRequestProto proto) {
+ public RegisterApplicationMasterRequestPBImpl(
+ RegisterApplicationMasterRequestProto proto) {
this.proto = proto;
viaProto = true;
}
@@ -71,6 +88,30 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM
}
private void mergeLocalToBuilder() {
+ if (this.placementConstraints != null) {
+ addPlacementConstraintMap();
+ }
+ }
+
+ private void addPlacementConstraintMap() {
+ maybeInitBuilder();
+ builder.clearPlacementConstraints();
+ if (this.placementConstraints == null) {
+ return;
+ }
+ List<YarnProtos.PlacementConstraintMapEntryProto> protoList =
+ new ArrayList<>();
+ for (Map.Entry<Set<String>, PlacementConstraint> entry :
+ this.placementConstraints.entrySet()) {
+ protoList.add(
+ YarnProtos.PlacementConstraintMapEntryProto.newBuilder()
+ .addAllAllocationTags(entry.getKey())
+ .setPlacementConstraint(
+ new PlacementConstraintToProtoConverter(
+ entry.getValue()).convert())
+ .build());
+ }
+ builder.addAllPlacementConstraints(protoList);
}
private void mergeLocalToProto() {
@@ -90,7 +131,8 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM
@Override
public String getHost() {
- RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ RegisterApplicationMasterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
return p.getHost();
}
@@ -106,7 +148,8 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM
@Override
public int getRpcPort() {
- RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ RegisterApplicationMasterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
return p.getRpcPort();
}
@@ -118,7 +161,8 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM
@Override
public String getTrackingUrl() {
- RegisterApplicationMasterRequestProtoOrBuilder p = viaProto ? proto : builder;
+ RegisterApplicationMasterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
return p.getTrackingUrl();
}
@@ -131,4 +175,50 @@ public class RegisterApplicationMasterRequestPBImpl extends RegisterApplicationM
}
builder.setTrackingUrl(url);
}
-}
+
+ private void initPlacementConstraintMap() {
+ if (this.placementConstraints != null) {
+ return;
+ }
+ RegisterApplicationMasterRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<YarnProtos.PlacementConstraintMapEntryProto> pcmList =
+ p.getPlacementConstraintsList();
+ this.placementConstraints = new HashMap<>();
+ for (YarnProtos.PlacementConstraintMapEntryProto e : pcmList) {
+ this.placementConstraints.put(
+ new HashSet<>(e.getAllocationTagsList()),
+ new PlacementConstraintFromProtoConverter(
+ e.getPlacementConstraint()).convert());
+ }
+ }
+
+ @Override
+ public Map<Set<String>, PlacementConstraint> getPlacementConstraints() {
+ initPlacementConstraintMap();
+ return this.placementConstraints;
+ }
+
+ @Override
+ public void setPlacementConstraints(
+ Map<Set<String>, PlacementConstraint> constraints) {
+ maybeInitBuilder();
+ if (constraints == null) {
+ builder.clearPlacementConstraints();
+ } else {
+ removeEmptyKeys(constraints);
+ }
+ this.placementConstraints = constraints;
+ }
+
+ private void removeEmptyKeys(
+ Map<Set<String>, PlacementConstraint> constraintMap) {
+ Iterator<Set<String>> iter = constraintMap.keySet().iterator();
+ while (iter.hasNext()) {
+ Set<String> aTags = iter.next();
+ if (aTags.size() == 0) {
+ iter.remove();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24deea4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
index 8694651..ebd66af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/BasePBImplRecordsTest.java
@@ -22,12 +22,19 @@ import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
import org.junit.Assert;
import java.lang.reflect.*;
import java.nio.ByteBuffer;
import java.util.*;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints
+ .PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+
/**
* Generic helper class to validate protocol records.
*/
@@ -85,6 +92,10 @@ public class BasePBImplRecordsTest {
ByteBuffer buff = ByteBuffer.allocate(4);
rand.nextBytes(buff.array());
return buff;
+ } else if (type.equals(PlacementConstraint.class)) {
+ PlacementConstraint.AbstractConstraint sConstraintExpr =
+ targetIn(NODE, allocationTag("foo"));
+ ret = PlacementConstraints.build(sConstraintExpr);
}
} else if (type instanceof ParameterizedType) {
ParameterizedType pt = (ParameterizedType)type;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/19] hadoop git commit: YARN-6593. [API] Introduce Placement
Constraint object. (Konstantinos Karanasos via wangda)
Posted by as...@apache.org.
YARN-6593. [API] Introduce Placement Constraint object. (Konstantinos Karanasos via wangda)
Change-Id: Id00edb7185fdf01cce6e40f920cac3585f8cbe9c
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90ac4f89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90ac4f89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90ac4f89
Branch: refs/heads/YARN-6592
Commit: 90ac4f89792ee6b6496118321fa9b566f40e53a9
Parents: 0940e4f
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Aug 3 14:03:55 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Nov 17 10:43:44 2017 -0800
----------------------------------------------------------------------
.../yarn/api/resource/PlacementConstraint.java | 567 +++++++++++++++++++
.../yarn/api/resource/PlacementConstraints.java | 286 ++++++++++
.../hadoop/yarn/api/resource/package-info.java | 23 +
.../src/main/proto/yarn_protos.proto | 55 ++
.../api/resource/TestPlacementConstraints.java | 106 ++++
.../PlacementConstraintFromProtoConverter.java | 116 ++++
.../pb/PlacementConstraintToProtoConverter.java | 174 ++++++
.../apache/hadoop/yarn/api/pb/package-info.java | 23 +
.../yarn/api/records/impl/pb/ProtoUtils.java | 27 +
.../PlacementConstraintTransformations.java | 209 +++++++
.../hadoop/yarn/api/resource/package-info.java | 23 +
.../TestPlacementConstraintPBConversion.java | 195 +++++++
.../TestPlacementConstraintTransformations.java | 183 ++++++
13 files changed, 1987 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
new file mode 100644
index 0000000..f0e3982
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraint.java
@@ -0,0 +1,567 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * {@code PlacementConstraint} represents a placement constraint for a resource
+ * allocation.
+ */
+@Public
+@Unstable
+public class PlacementConstraint {
+
+ /**
+ * The constraint expression tree.
+ */
+ private AbstractConstraint constraintExpr;
+
+ public PlacementConstraint(AbstractConstraint constraintExpr) {
+ this.constraintExpr = constraintExpr;
+ }
+
+ /**
+ * Get the constraint expression of the placement constraint.
+ *
+ * @return the constraint expression
+ */
+ public AbstractConstraint getConstraintExpr() {
+ return constraintExpr;
+ }
+
+ /**
+ * Interface used to enable the elements of the constraint tree to be visited.
+ */
+ @Private
+ public interface Visitable {
+ /**
+ * Visitor pattern.
+ *
+ * @param visitor visitor to be used
+ * @param <T> defines the type that the visitor will use and the return type
+ * of the accept.
+ * @return the result of visiting a given object.
+ */
+ <T> T accept(Visitor<T> visitor);
+
+ }
+
+ /**
+ * Visitor API for a constraint tree.
+ *
+ * @param <T> determines the return type of the visit methods.
+ */
+ @Private
+ public interface Visitor<T> {
+ T visit(SingleConstraint constraint);
+
+ T visit(TargetExpression target);
+
+ T visit(TargetConstraint constraint);
+
+ T visit(CardinalityConstraint constraint);
+
+ T visit(And constraint);
+
+ T visit(Or constraint);
+
+ T visit(DelayedOr constraint);
+
+ T visit(TimedPlacementConstraint constraint);
+ }
+
+ /**
+ * Abstract class that acts as the superclass of all placement constraint
+ * classes.
+ */
+ public abstract static class AbstractConstraint implements Visitable {
+ public PlacementConstraint build() {
+ return new PlacementConstraint(this);
+ }
+ }
+
+ static final String NODE_SCOPE = "node";
+ static final String RACK_SCOPE = "rack";
+
+ /**
+ * Consider a set of nodes N that belongs to the scope specified in the
+ * constraint. If the target expressions are satisfied at least minCardinality
+ * times and at most max-cardinality times in the node set N, then the
+ * constraint is satisfied.
+ *
+ * For example, a constraint of the form {@code {RACK, 2, 10,
+ * allocationTag("zk")}}, requires an allocation to be placed within a rack
+ * that has at least 2 and at most 10 other allocations with tag "zk".
+ */
+ public static class SingleConstraint extends AbstractConstraint {
+ private String scope;
+ private int minCardinality;
+ private int maxCardinality;
+ private Set<TargetExpression> targetExpressions;
+
+ public SingleConstraint(String scope, int minCardinality,
+ int maxCardinality, Set<TargetExpression> targetExpressions) {
+ this.scope = scope;
+ this.minCardinality = minCardinality;
+ this.maxCardinality = maxCardinality;
+ this.targetExpressions = targetExpressions;
+ }
+
+ public SingleConstraint(String scope, int minC, int maxC,
+ TargetExpression... targetExpressions) {
+ this(scope, minC, maxC, new HashSet<>(Arrays.asList(targetExpressions)));
+ }
+
+ /**
+ * Get the scope of the constraint.
+ *
+ * @return the scope of the constraint
+ */
+ public String getScope() {
+ return scope;
+ }
+
+ /**
+ * Get the minimum cardinality of the constraint.
+ *
+ * @return the minimum cardinality of the constraint
+ */
+ public int getMinCardinality() {
+ return minCardinality;
+ }
+
+ /**
+ * Get the maximum cardinality of the constraint.
+ *
+ * @return the maximum cardinality of the constraint
+ */
+ public int getMaxCardinality() {
+ return maxCardinality;
+ }
+
+ /**
+ * Get the target expressions of the constraint.
+ *
+ * @return the set of target expressions
+ */
+ public Set<TargetExpression> getTargetExpressions() {
+ return targetExpressions;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class representing the target expressions that are used in placement
+ * constraints. They might refer to expressions on node attributes, allocation
+ * tags, or be self-targets (referring to the allocation to which the
+ * constraint is attached).
+ */
+ public static class TargetExpression implements Visitable {
+ /**
+ * Enum specifying the type of the target expression.
+ */
+ public enum TargetType {
+ NODE_ATTRIBUTE, ALLOCATION_TAG, SELF
+ }
+
+ private TargetType targetType;
+ private String targetKey;
+ private Set<String> targetValues;
+
+ public TargetExpression(TargetType targetType, String targetKey,
+ Set<String> targetValues) {
+ this.targetType = targetType;
+ this.targetKey = targetKey;
+ this.targetValues = targetValues;
+ }
+
+ public TargetExpression(TargetType targetType) {
+ this(targetType, null, new HashSet<>());
+ }
+
+ public TargetExpression(TargetType targetType, String targetKey,
+ String... targetValues) {
+ this(targetType, targetKey, new HashSet<>(Arrays.asList(targetValues)));
+ }
+
+ /**
+ * Get the type of the target expression.
+ *
+ * @return the type of the target expression
+ */
+ public TargetType getTargetType() {
+ return targetType;
+ }
+
+ /**
+ * Get the key of the target expression.
+ *
+ * @return the key of the target expression
+ */
+ public String getTargetKey() {
+ return targetKey;
+ }
+
+ /**
+ * Get the set of values of the target expression.
+ *
+ * @return the set of values of the target expression
+ */
+ public Set<String> getTargetValues() {
+ return targetValues;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = targetType != null ? targetType.hashCode() : 0;
+ result = 31 * result + (targetKey != null ? targetKey.hashCode() : 0);
+ result =
+ 31 * result + (targetValues != null ? targetValues.hashCode() : 0);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null) {
+ return false;
+ }
+ if (!(o instanceof TargetExpression)) {
+ return false;
+ }
+
+ TargetExpression that = (TargetExpression) o;
+ if (targetType != that.targetType) {
+ return false;
+ }
+ if (targetKey != null ? !targetKey.equals(that.targetKey)
+ : that.targetKey != null) {
+ return false;
+ }
+ return targetValues != null ? targetValues.equals(that.targetValues)
+ : that.targetValues == null;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a target constraint. Such a constraint requires an
+ * allocation to be placed within a scope that satisfies some specified
+ * expressions on node attributes and allocation tags.
+ *
+ * It is a specialized version of the {@link SingleConstraint}, where the
+ * minimum and the maximum cardinalities take specific values based on the
+ * {@link TargetOperator} used.
+ */
+ public static class TargetConstraint extends AbstractConstraint {
+ enum TargetOperator {
+ IN, NOT_IN
+ }
+
+ private TargetOperator op;
+ private String scope;
+ private Set<TargetExpression> targetExpressions;
+
+ public TargetConstraint(TargetOperator op, String scope,
+ Set<TargetExpression> targetExpressions) {
+ this.op = op;
+ this.scope = scope;
+ this.targetExpressions = targetExpressions;
+ }
+
+ /**
+ * Get the target operator of the constraint.
+ *
+ * @return the target operator
+ */
+ public TargetOperator getOp() {
+ return op;
+ }
+
+ /**
+ * Get the scope of the constraint.
+ *
+ * @return the scope of the constraint
+ */
+ public String getScope() {
+ return scope;
+ }
+
+ /**
+ * Get the set of target expressions.
+ *
+ * @return the set of target expressions
+ */
+ public Set<TargetExpression> getTargetExpressions() {
+ return targetExpressions;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a cardinality constraint. Such a constraint the
+ * number of allocations within a given scope to some minimum and maximum
+ * values.
+ *
+ * It is a specialized version of the {@link SingleConstraint}, where the
+ * target is self (i.e., the allocation to which the constraint is attached).
+ */
+ public static class CardinalityConstraint extends AbstractConstraint {
+ private String scope;
+ private int minCardinality;
+ private int maxCardinality;
+
+ public CardinalityConstraint(String scope, int minCardinality,
+ int maxCardinality) {
+ this.scope = scope;
+ this.minCardinality = minCardinality;
+ this.maxCardinality = maxCardinality;
+ }
+
+ /**
+ * Get the scope of the constraint.
+ *
+ * @return the scope of the constraint
+ */
+ public String getScope() {
+ return scope;
+ }
+
+ /**
+ * Get the minimum cardinality of the constraint.
+ *
+ * @return the minimum cardinality of the constraint
+ */
+ public int getMinCardinality() {
+ return minCardinality;
+ }
+
+ /**
+ * Get the maximum cardinality of the constraint.
+ *
+ * @return the maximum cardinality of the constraint
+ */
+ public int getMaxCardinality() {
+ return maxCardinality;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents composite constraints, which comprise other
+ * constraints, forming a constraint tree.
+ *
+ * @param <R> the type of constraints that are used as children of the
+ * specific composite constraint
+ */
+ public abstract static class CompositeConstraint<R extends Visitable>
+ extends AbstractConstraint {
+
+ /**
+ * Get the children of this composite constraint.
+ *
+ * @return the children of the composite constraint
+ */
+ public abstract List<R> getChildren();
+ }
+
+ /**
+ * Class that represents a composite constraint that is a conjunction of other
+ * constraints.
+ */
+ public static class And extends CompositeConstraint<AbstractConstraint> {
+ private List<AbstractConstraint> children;
+
+ public And(List<AbstractConstraint> children) {
+ this.children = children;
+ }
+
+ public And(AbstractConstraint... children) {
+ this(Arrays.asList(children));
+ }
+
+ @Override
+ public List<AbstractConstraint> getChildren() {
+ return children;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a composite constraint that is a disjunction of other
+ * constraints.
+ */
+ public static class Or extends CompositeConstraint<AbstractConstraint> {
+ private List<AbstractConstraint> children;
+
+ public Or(List<AbstractConstraint> children) {
+ this.children = children;
+ }
+
+ public Or(AbstractConstraint... children) {
+ this(Arrays.asList(children));
+ }
+
+ @Override
+ public List<AbstractConstraint> getChildren() {
+ return children;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Class that represents a composite constraint that comprises a list of timed
+ * placement constraints (see {@link TimedPlacementConstraint}). The scheduler
+ * should try to satisfy first the first timed child constraint within the
+ * specified time window. If this is not possible, it should attempt to
+ * satisfy the second, and so on.
+ */
+ public static class DelayedOr
+ extends CompositeConstraint<TimedPlacementConstraint> {
+ private List<TimedPlacementConstraint> children = new ArrayList<>();
+
+ public DelayedOr(List<TimedPlacementConstraint> children) {
+ this.children = children;
+ }
+
+ public DelayedOr(TimedPlacementConstraint... children) {
+ this(Arrays.asList(children));
+ }
+
+ @Override
+ public List<TimedPlacementConstraint> getChildren() {
+ return children;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+
+ /**
+ * Represents a timed placement constraint that has to be satisfied within a
+ * time window.
+ */
+ public static class TimedPlacementConstraint implements Visitable {
+ /**
+ * The unit of scheduling delay.
+ */
+ public enum DelayUnit {
+ MILLISECONDS, OPPORTUNITIES
+ }
+
+ private AbstractConstraint constraint;
+ private long schedulingDelay;
+ private DelayUnit delayUnit;
+
+ public TimedPlacementConstraint(AbstractConstraint constraint,
+ long schedulingDelay, DelayUnit delayUnit) {
+ this.constraint = constraint;
+ this.schedulingDelay = schedulingDelay;
+ this.delayUnit = delayUnit;
+ }
+
+ public TimedPlacementConstraint(AbstractConstraint constraint,
+ long schedulingDelay) {
+ this(constraint, schedulingDelay, DelayUnit.MILLISECONDS);
+ }
+
+ public TimedPlacementConstraint(AbstractConstraint constraint) {
+ this(constraint, Long.MAX_VALUE, DelayUnit.MILLISECONDS);
+ }
+
+ /**
+ * Get the constraint that has to be satisfied within the time window.
+ *
+ * @return the constraint to be satisfied
+ */
+ public AbstractConstraint getConstraint() {
+ return constraint;
+ }
+
+ /**
+ * Sets the constraint that has to be satisfied within the time window.
+ *
+ * @param constraint the constraint to be satisfied
+ */
+ public void setConstraint(AbstractConstraint constraint) {
+ this.constraint = constraint;
+ }
+
+ /**
+ * Get the scheduling delay value that determines the time window within
+ * which the constraint has to be satisfied.
+ *
+ * @return the value of the scheduling delay
+ */
+ public long getSchedulingDelay() {
+ return schedulingDelay;
+ }
+
+ /**
+ * The unit of the scheduling delay.
+ *
+ * @return the unit of the delay
+ */
+ public DelayUnit getDelayUnit() {
+ return delayUnit;
+ }
+
+ @Override
+ public <T> T accept(Visitor<T> visitor) {
+ return visitor.visit(this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
new file mode 100644
index 0000000..8e84280
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraints.java
@@ -0,0 +1,286 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+
+/**
+ * This class contains various static methods for the applications to create
+ * placement constraints (see also {@link PlacementConstraint}).
+ */
+@Public
+@Unstable
+public final class PlacementConstraints {
+
+ // Suppresses default constructor, ensuring non-instantiability.
+ private PlacementConstraints() {
+ }
+
+ // Creation of simple constraints.
+
+ public static final String NODE = PlacementConstraint.NODE_SCOPE;
+ public static final String RACK = PlacementConstraint.RACK_SCOPE;
+
+ /**
+ * Creates a constraint that requires allocations to be placed on nodes that
+ * satisfy all target expressions within the given scope (e.g., node or rack).
+ *
+ * For example, {@code targetIn(RACK, allocationTag("hbase-m"))}, allows
+ * allocations on nodes that belong to a rack that has at least one tag with
+ * value "hbase-m".
+ *
+ * @param scope the scope within which the target expressions should be
+ * satisfied
+ * @param targetExpressions the expressions that need to be satisfied within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint targetIn(String scope,
+ TargetExpression... targetExpressions) {
+ return new SingleConstraint(scope, 1, Integer.MAX_VALUE, targetExpressions);
+ }
+
+ /**
+ * Creates a constraint that requires allocations to be placed on nodes that
+ * belong to a scope (e.g., node or rack) that does not satisfy any of the
+ * target expressions.
+ *
+ * @param scope the scope within which the target expressions should not be
+ * true
+ * @param targetExpressions the expressions that need to not be true within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint targetNotIn(String scope,
+ TargetExpression... targetExpressions) {
+ return new SingleConstraint(scope, 0, 0, targetExpressions);
+ }
+
+ /**
+ * Creates a constraint that restricts the number of allocations within a
+ * given scope (e.g., node or rack).
+ *
+ * For example, {@code cardinality(NODE, 3, 10)}, restricts the number of
+ * allocations per node to be no less than 3 and no more than 10.
+ *
+ * @param scope the scope of the constraint
+ * @param minCardinality determines the minimum number of allocations within
+ * the scope
+ * @param maxCardinality determines the maximum number of allocations within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint cardinality(String scope, int minCardinality,
+ int maxCardinality) {
+ return new SingleConstraint(scope, minCardinality, maxCardinality,
+ PlacementTargets.self());
+ }
+
+ /**
+ * Similar to {@link #cardinality(String, int, int)}, but determines only the
+ * minimum cardinality (the maximum cardinality is unbound).
+ *
+ * @param scope the scope of the constraint
+ * @param minCardinality determines the minimum number of allocations within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint minCardinality(String scope,
+ int minCardinality) {
+ return cardinality(scope, minCardinality, Integer.MAX_VALUE);
+ }
+
+ /**
+ * Similar to {@link #cardinality(String, int, int)}, but determines only the
+ * maximum cardinality (the minimum can be as low as 0).
+ *
+ * @param scope the scope of the constraint
+ * @param maxCardinality determines the maximum number of allocations within
+ * the scope
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint maxCardinality(String scope,
+ int maxCardinality) {
+ return cardinality(scope, 0, maxCardinality);
+ }
+
+ /**
+ * This constraint generalizes the cardinality and target constraints.
+ *
+ * Consider a set of nodes N that belongs to the scope specified in the
+ * constraint. If the target expressions are satisfied at least minCardinality
+ * times and at most max-cardinality times in the node set N, then the
+ * constraint is satisfied.
+ *
+ * For example, {@code targetCardinality(RACK, 2, 10, allocationTag("zk"))},
+ * requires an allocation to be placed within a rack that has at least 2 and
+ * at most 10 other allocations with tag "zk".
+ *
+ * @param scope the scope of the constraint
+ * @param minCardinality the minimum number of times the target expressions
+ * have to be satisfied with the given scope
+ * @param maxCardinality the maximum number of times the target expressions
+ * have to be satisfied with the given scope
+ * @param targetExpressions the target expressions
+ * @return the resulting placement constraint
+ */
+ public static AbstractConstraint targetCardinality(String scope,
+ int minCardinality, int maxCardinality,
+ TargetExpression... targetExpressions) {
+ return new SingleConstraint(scope, minCardinality, maxCardinality,
+ targetExpressions);
+ }
+
+ // Creation of target expressions to be used in simple constraints.
+
+ /**
+ * Class with static methods for constructing target expressions to be used in
+ * placement constraints.
+ */
+ public static class PlacementTargets {
+
+ /**
+ * Constructs a target expression on a node attribute. It is satisfied if
+ * the specified node attribute has one of the specified values.
+ *
+ * @param attributeKey the name of the node attribute
+ * @param attributeValues the set of values that the attribute should take
+ * values from
+ * @return the resulting expression on the node attribute
+ */
+ public static TargetExpression nodeAttribute(String attributeKey,
+ String... attributeValues) {
+ return new TargetExpression(TargetType.NODE_ATTRIBUTE, attributeKey,
+ attributeValues);
+ }
+
+ /**
+ * Constructs a target expression on an allocation tag. It is satisfied if
+ * the there are allocations with one of the given tags.
+ *
+ * @param allocationTags the set of tags that the attribute should take
+ * values from
+ * @return the resulting expression on the allocation tags
+ */
+ public static TargetExpression allocationTag(String... allocationTags) {
+ return new TargetExpression(TargetType.ALLOCATION_TAG, null,
+ allocationTags);
+ }
+
+ /**
+ * The default target expression that uses as target the allocation that
+ * specifies the constraint.
+ *
+ * @return the self-target
+ */
+ public static TargetExpression self() {
+ return new TargetExpression(TargetType.SELF);
+ }
+ }
+
+ // Creation of compound constraints.
+
+ /**
+ * A conjunction of constraints.
+ *
+ * @param children the children constraints that should all be satisfied
+ * @return the resulting placement constraint
+ */
+ public static And and(AbstractConstraint... children) {
+ return new And(children);
+ }
+
+ /**
+ * A disjunction of constraints.
+ *
+ * @param children the children constraints, one of which should be satisfied
+ * @return the resulting placement constraint
+ */
+ public static Or or(AbstractConstraint... children) {
+ return new Or(children);
+ }
+
+ /**
+ * Creates a composite constraint that includes a list of timed placement
+ * constraints. The scheduler should try to satisfy first the first timed
+ * child constraint within the specified time window. If this is not possible,
+ * it should attempt to satisfy the second, and so on.
+ *
+ * @param children the timed children constraints
+ * @return the resulting composite constraint
+ */
+ public static DelayedOr delayedOr(TimedPlacementConstraint... children) {
+ return new DelayedOr(children);
+ }
+
+ // Creation of timed constraints to be used in a DELAYED_OR constraint.
+
+ /**
+ * Creates a placement constraint that has to be satisfied within a time
+ * window.
+ *
+ * @param constraint the placement constraint
+ * @param delay the length of the time window within which the constraint has
+ * to be satisfied
+ * @param timeUnit the unit of time of the time window
+ * @return the resulting timed placement constraint
+ */
+ public static TimedPlacementConstraint timedClockConstraint(
+ AbstractConstraint constraint, long delay, TimeUnit timeUnit) {
+ return new TimedPlacementConstraint(constraint, timeUnit.toMillis(delay),
+ TimedPlacementConstraint.DelayUnit.MILLISECONDS);
+ }
+
+ /**
+ * Creates a placement constraint that has to be satisfied within a number of
+ * placement opportunities (invocations of the scheduler).
+ *
+ * @param constraint the placement constraint
+ * @param delay the number of scheduling opportunities within which the
+ * constraint has to be satisfied
+ * @return the resulting timed placement constraint
+ */
+ public static TimedPlacementConstraint timedOpportunitiesConstraint(
+ AbstractConstraint constraint, long delay) {
+ return new TimedPlacementConstraint(constraint, delay,
+ TimedPlacementConstraint.DelayUnit.OPPORTUNITIES);
+ }
+
+ /**
+ * Creates a {@link PlacementConstraint} given a constraint expression.
+ *
+ * @param constraintExpr the constraint expression
+ * @return the placement constraint
+ */
+ public static PlacementConstraint build(AbstractConstraint constraintExpr) {
+ return constraintExpr.build();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
new file mode 100644
index 0000000..660dc02
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * API related to resources.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.api.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 7769c48..3739d94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -566,6 +566,61 @@ enum SignalContainerCommandProto {
FORCEFUL_SHUTDOWN = 3;
}
+////////////////////////////////////////////////////////////////////////
+////// Placement constraints ///////////////////////////////////////////
+////////////////////////////////////////////////////////////////////////
+
+message PlacementConstraintProto {
+ optional SimplePlacementConstraintProto simpleConstraint = 1;
+ optional CompositePlacementConstraintProto compositeConstraint = 2;
+}
+
+message SimplePlacementConstraintProto {
+ required string scope = 1;
+ repeated PlacementConstraintTargetProto targetExpressions = 2;
+ optional int32 minCardinality = 3;
+ optional int32 maxCardinality = 4;
+}
+
+message PlacementConstraintTargetProto {
+ enum TargetType {
+ NODE_ATTRIBUTE = 1;
+ ALLOCATION_TAG = 2;
+ SELF = 3;
+ }
+
+ required TargetType targetType = 1;
+ optional string targetKey = 2;
+ repeated string targetValues = 3;
+}
+
+message TimedPlacementConstraintProto {
+ enum DelayUnit {
+ MILLISECONDS = 1;
+ OPPORTUNITIES = 2;
+ }
+
+ required PlacementConstraintProto placementConstraint = 1;
+ required int64 schedulingDelay = 2;
+ optional DelayUnit delayUnit = 3 [ default = MILLISECONDS ];
+}
+
+message CompositePlacementConstraintProto {
+ enum CompositeType {
+ // All children constraints have to be satisfied.
+ AND = 1;
+ // One of the children constraints has to be satisfied.
+ OR = 2;
+ // Attempt to satisfy the first child constraint for delays[0] units (e.g.,
+ // millisec or heartbeats). If this fails, try to satisfy the second child
+ // constraint for delays[1] units and so on.
+ DELAYED_OR = 3;
+ }
+
+ required CompositeType compositeType = 1;
+ repeated PlacementConstraintProto childConstraints = 2;
+ repeated TimedPlacementConstraintProto timedChildConstraints = 3;
+}
////////////////////////////////////////////////////////////////////////
////// From reservation_protocol /////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
new file mode 100644
index 0000000..e25d477
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraints.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.and;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetNotIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.nodeAttribute;
+
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression.TargetType;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test class for the various static methods in
+ * {@link org.apache.hadoop.yarn.api.resource.PlacementConstraints}.
+ */
+public class TestPlacementConstraints {
+
+ @Test
+ public void testNodeAffinityToTag() {
+ AbstractConstraint constraintExpr =
+ targetIn(NODE, allocationTag("hbase-m"));
+
+ SingleConstraint sConstraint = (SingleConstraint) constraintExpr;
+ Assert.assertEquals(NODE, sConstraint.getScope());
+ Assert.assertEquals(1, sConstraint.getMinCardinality());
+ Assert.assertEquals(Integer.MAX_VALUE, sConstraint.getMaxCardinality());
+
+ Assert.assertEquals(1, sConstraint.getTargetExpressions().size());
+ TargetExpression tExpr =
+ sConstraint.getTargetExpressions().iterator().next();
+ Assert.assertNull(tExpr.getTargetKey());
+ Assert.assertEquals(TargetType.ALLOCATION_TAG, tExpr.getTargetType());
+ Assert.assertEquals(1, tExpr.getTargetValues().size());
+ Assert.assertEquals("hbase-m", tExpr.getTargetValues().iterator().next());
+
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+ Assert.assertNotNull(constraint.getConstraintExpr());
+ }
+
+ @Test
+ public void testNodeAntiAffinityToAttribute() {
+ AbstractConstraint constraintExpr =
+ targetNotIn(NODE, nodeAttribute("java", "1.8"));
+
+ SingleConstraint sConstraint = (SingleConstraint) constraintExpr;
+ Assert.assertEquals(NODE, sConstraint.getScope());
+ Assert.assertEquals(0, sConstraint.getMinCardinality());
+ Assert.assertEquals(0, sConstraint.getMaxCardinality());
+
+ Assert.assertEquals(1, sConstraint.getTargetExpressions().size());
+ TargetExpression tExpr =
+ sConstraint.getTargetExpressions().iterator().next();
+ Assert.assertEquals("java", tExpr.getTargetKey());
+ Assert.assertEquals(TargetType.NODE_ATTRIBUTE, tExpr.getTargetType());
+ Assert.assertEquals(1, tExpr.getTargetValues().size());
+ Assert.assertEquals("1.8", tExpr.getTargetValues().iterator().next());
+ }
+
+ @Test
+ public void testAndConstraint() {
+ AbstractConstraint constraintExpr =
+ and(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
+ targetCardinality(RACK, 2, 10, allocationTag("zk")));
+
+ And andExpr = (And) constraintExpr;
+ Assert.assertEquals(3, andExpr.getChildren().size());
+ SingleConstraint sConstr = (SingleConstraint) andExpr.getChildren().get(0);
+ TargetExpression tExpr = sConstr.getTargetExpressions().iterator().next();
+ Assert.assertEquals("spark", tExpr.getTargetValues().iterator().next());
+
+ sConstr = (SingleConstraint) andExpr.getChildren().get(1);
+ Assert.assertEquals(0, sConstr.getMinCardinality());
+ Assert.assertEquals(3, sConstr.getMaxCardinality());
+
+ sConstr = (SingleConstraint) andExpr.getChildren().get(2);
+ Assert.assertEquals(2, sConstr.getMinCardinality());
+ Assert.assertEquals(10, sConstr.getMaxCardinality());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java
new file mode 100644
index 0000000..926b6fa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintFromProtoConverter.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.pb;
+
+import static org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType.AND;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
+
+/**
+ * {@code PlacementConstraintFromProtoConverter} generates an
+ * {@link PlacementConstraint.AbstractConstraint} given a
+ * {@link PlacementConstraintProto}.
+ */
+@Private
+public class PlacementConstraintFromProtoConverter {
+
+ private PlacementConstraintProto constraintProto;
+
+ public PlacementConstraintFromProtoConverter(
+ PlacementConstraintProto constraintProto) {
+ this.constraintProto = constraintProto;
+ }
+
+ public PlacementConstraint convert() {
+ return new PlacementConstraint(convert(constraintProto));
+ }
+
+ private AbstractConstraint convert(PlacementConstraintProto proto) {
+ return proto.hasSimpleConstraint() ? convert(proto.getSimpleConstraint())
+ : convert(proto.getCompositeConstraint());
+ }
+
+ private SingleConstraint convert(SimplePlacementConstraintProto proto) {
+ Set<TargetExpression> targets = new HashSet<>();
+ for (PlacementConstraintTargetProto tp : proto.getTargetExpressionsList()) {
+ targets.add(convert(tp));
+ }
+
+ return new SingleConstraint(proto.getScope(), proto.getMinCardinality(),
+ proto.getMaxCardinality(), targets);
+ }
+
+ private TargetExpression convert(PlacementConstraintTargetProto proto) {
+ return new TargetExpression(
+ ProtoUtils.convertFromProtoFormat(proto.getTargetType()),
+ proto.hasTargetKey() ? proto.getTargetKey() : null,
+ new HashSet<>(proto.getTargetValuesList()));
+ }
+
+ private AbstractConstraint convert(CompositePlacementConstraintProto proto) {
+ switch (proto.getCompositeType()) {
+ case AND:
+ case OR:
+ List<AbstractConstraint> children = new ArrayList<>();
+ for (PlacementConstraintProto cp : proto.getChildConstraintsList()) {
+ children.add(convert(cp));
+ }
+ return (proto.getCompositeType() == AND) ? new And(children)
+ : new Or(children);
+ case DELAYED_OR:
+ List<TimedPlacementConstraint> tChildren = new ArrayList<>();
+ for (TimedPlacementConstraintProto cp : proto
+ .getTimedChildConstraintsList()) {
+ tChildren.add(convert(cp));
+ }
+ return new DelayedOr(tChildren);
+ default:
+ throw new YarnRuntimeException(
+ "Encountered unexpected type of composite constraint.");
+ }
+ }
+
+ private TimedPlacementConstraint convert(
+ TimedPlacementConstraintProto proto) {
+ AbstractConstraint pConstraint = convert(proto.getPlacementConstraint());
+
+ return new TimedPlacementConstraint(pConstraint, proto.getSchedulingDelay(),
+ ProtoUtils.convertFromProtoFormat(proto.getDelayUnit()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
new file mode 100644
index 0000000..7816e18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/PlacementConstraintToProtoConverter.java
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
+
+import com.google.protobuf.GeneratedMessage;
+
+/**
+ * {@code PlacementConstraintToProtoConverter} generates a
+ * {@link PlacementConstraintProto} given a
+ * {@link PlacementConstraint.AbstractConstraint}.
+ */
+@Private
+public class PlacementConstraintToProtoConverter
+ implements PlacementConstraint.Visitor<GeneratedMessage> {
+
+ private PlacementConstraint placementConstraint;
+
+ public PlacementConstraintToProtoConverter(
+ PlacementConstraint placementConstraint) {
+ this.placementConstraint = placementConstraint;
+ }
+
+ public PlacementConstraintProto convert() {
+ return (PlacementConstraintProto) placementConstraint.getConstraintExpr()
+ .accept(this);
+ }
+
+ @Override
+ public GeneratedMessage visit(SingleConstraint constraint) {
+ SimplePlacementConstraintProto.Builder sb =
+ SimplePlacementConstraintProto.newBuilder();
+
+ if (constraint.getScope() != null) {
+ sb.setScope(constraint.getScope());
+ }
+ sb.setMinCardinality(constraint.getMinCardinality());
+ sb.setMaxCardinality(constraint.getMaxCardinality());
+ if (constraint.getTargetExpressions() != null) {
+ for (TargetExpression target : constraint.getTargetExpressions()) {
+ sb.addTargetExpressions(
+ (PlacementConstraintTargetProto) target.accept(this));
+ }
+
+ }
+ SimplePlacementConstraintProto sProto = sb.build();
+
+ // Wrap around PlacementConstraintProto object.
+ PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder();
+ pb.setSimpleConstraint(sProto);
+ return pb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(TargetExpression target) {
+ PlacementConstraintTargetProto.Builder tb =
+ PlacementConstraintTargetProto.newBuilder();
+
+ tb.setTargetType(ProtoUtils.convertToProtoFormat(target.getTargetType()));
+ if (target.getTargetKey() != null) {
+ tb.setTargetKey(target.getTargetKey());
+ }
+ if (target.getTargetValues() != null) {
+ tb.addAllTargetValues(target.getTargetValues());
+ }
+ return tb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(TargetConstraint constraint) {
+ throw new YarnRuntimeException("Unexpected TargetConstraint found.");
+ }
+
+ @Override
+ public GeneratedMessage visit(CardinalityConstraint constraint) {
+ throw new YarnRuntimeException("Unexpected CardinalityConstraint found.");
+ }
+
+ private GeneratedMessage visitAndOr(
+ CompositeConstraint<AbstractConstraint> composite, CompositeType type) {
+ CompositePlacementConstraintProto.Builder cb =
+ CompositePlacementConstraintProto.newBuilder();
+
+ cb.setCompositeType(type);
+
+ for (AbstractConstraint c : composite.getChildren()) {
+ cb.addChildConstraints((PlacementConstraintProto) c.accept(this));
+ }
+ CompositePlacementConstraintProto cProto = cb.build();
+
+ // Wrap around PlacementConstraintProto object.
+ PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder();
+ pb.setCompositeConstraint(cProto);
+ return pb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(And constraint) {
+ return visitAndOr(constraint, CompositeType.AND);
+ }
+
+ @Override
+ public GeneratedMessage visit(Or constraint) {
+ return visitAndOr(constraint, CompositeType.OR);
+ }
+
+ @Override
+ public GeneratedMessage visit(DelayedOr constraint) {
+ CompositePlacementConstraintProto.Builder cb =
+ CompositePlacementConstraintProto.newBuilder();
+
+ cb.setCompositeType(CompositeType.DELAYED_OR);
+
+ for (TimedPlacementConstraint c : constraint.getChildren()) {
+ cb.addTimedChildConstraints(
+ (TimedPlacementConstraintProto) c.accept(this));
+ }
+ CompositePlacementConstraintProto cProto = cb.build();
+
+ // Wrap around PlacementConstraintProto object.
+ PlacementConstraintProto.Builder pb = PlacementConstraintProto.newBuilder();
+ pb.setCompositeConstraint(cProto);
+ return pb.build();
+ }
+
+ @Override
+ public GeneratedMessage visit(TimedPlacementConstraint constraint) {
+ TimedPlacementConstraintProto.Builder tb =
+ TimedPlacementConstraintProto.newBuilder();
+
+ tb.setDelayUnit(ProtoUtils.convertToProtoFormat(constraint.getDelayUnit()));
+ tb.setSchedulingDelay(constraint.getSchedulingDelay());
+ tb.setPlacementConstraint(
+ (PlacementConstraintProto) constraint.getConstraint().accept(this));
+
+ return tb.build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
new file mode 100644
index 0000000..18da80f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/pb/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * API related to protobuf objects that are not backed by PBImpl classes.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.api.pb;
+import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index 528cf8e..18075f52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -55,6 +55,8 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@@ -69,10 +71,12 @@ import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceVisibilityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintTargetProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ReservationRequestInterpreterProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.TimedPlacementConstraintProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryPolicyProto;
@@ -495,6 +499,29 @@ public class ProtoUtils {
}
return ret;
}
+
+ public static PlacementConstraintTargetProto.TargetType convertToProtoFormat(
+ TargetExpression.TargetType t) {
+ return PlacementConstraintTargetProto.TargetType.valueOf(t.name());
+ }
+
+ public static TargetExpression.TargetType convertFromProtoFormat(
+ PlacementConstraintTargetProto.TargetType t) {
+ return TargetExpression.TargetType.valueOf(t.name());
+ }
+
+ /*
+ * TimedPlacementConstraint.DelayUnit
+ */
+ public static TimedPlacementConstraintProto.DelayUnit convertToProtoFormat(
+ TimedPlacementConstraint.DelayUnit u) {
+ return TimedPlacementConstraintProto.DelayUnit.valueOf(u.name());
+ }
+
+ public static TimedPlacementConstraint.DelayUnit convertFromProtoFormat(
+ TimedPlacementConstraintProto.DelayUnit u) {
+ return TimedPlacementConstraint.DelayUnit.valueOf(u.name());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java
new file mode 100644
index 0000000..e9eda6f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/PlacementConstraintTransformations.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import java.util.ListIterator;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.And;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CompositeConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.DelayedOr;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint.TargetOperator;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetExpression;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TimedPlacementConstraint;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+/**
+ * This class contains inner classes that define transformation on a
+ * {@link PlacementConstraint} expression.
+ */
+@Private
+public class PlacementConstraintTransformations {
+
+ /**
+ * The default implementation of the {@link PlacementConstraint.Visitor} that
+ * does a traversal of the constraint tree, performing no action for the lead
+ * constraints.
+ */
+ public static class AbstractTransformer
+ implements PlacementConstraint.Visitor<AbstractConstraint> {
+
+ private PlacementConstraint placementConstraint;
+
+ public AbstractTransformer(PlacementConstraint placementConstraint) {
+ this.placementConstraint = placementConstraint;
+ }
+
+ /**
+ * This method performs the transformation of the
+ * {@link #placementConstraint}.
+ *
+ * @return the transformed placement constraint.
+ */
+ public PlacementConstraint transform() {
+ AbstractConstraint constraintExpr =
+ placementConstraint.getConstraintExpr();
+
+ // Visit the constraint tree to perform the transformation.
+ constraintExpr = constraintExpr.accept(this);
+
+ return new PlacementConstraint(constraintExpr);
+ }
+
+ @Override
+ public AbstractConstraint visit(SingleConstraint constraint) {
+ // Do nothing.
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(TargetExpression expression) {
+ // Do nothing.
+ return null;
+ }
+
+ @Override
+ public AbstractConstraint visit(TargetConstraint constraint) {
+ // Do nothing.
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(CardinalityConstraint constraint) {
+ // Do nothing.
+ return constraint;
+ }
+
+ private AbstractConstraint visitAndOr(
+ CompositeConstraint<AbstractConstraint> constraint) {
+ for (ListIterator<AbstractConstraint> iter =
+ constraint.getChildren().listIterator(); iter.hasNext();) {
+ AbstractConstraint child = iter.next();
+ child = child.accept(this);
+ iter.set(child);
+ }
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(And constraint) {
+ return visitAndOr(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(Or constraint) {
+ return visitAndOr(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(DelayedOr constraint) {
+ constraint.getChildren().forEach(
+ child -> child.setConstraint(child.getConstraint().accept(this)));
+ return constraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(TimedPlacementConstraint constraint) {
+ // Do nothing.
+ return null;
+ }
+ }
+
+ /**
+ * Visits a {@link PlacementConstraint} tree and substitutes each
+ * {@link TargetConstraint} and {@link CardinalityConstraint} with an
+ * equivalent {@link SingleConstraint}.
+ */
+ public static class SingleConstraintTransformer extends AbstractTransformer {
+
+ public SingleConstraintTransformer(PlacementConstraint constraint) {
+ super(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(TargetConstraint constraint) {
+ AbstractConstraint newConstraint;
+ if (constraint.getOp() == TargetOperator.IN) {
+ newConstraint = new SingleConstraint(constraint.getScope(), 1,
+ Integer.MAX_VALUE, constraint.getTargetExpressions());
+ } else if (constraint.getOp() == TargetOperator.NOT_IN) {
+ newConstraint = new SingleConstraint(constraint.getScope(), 0, 0,
+ constraint.getTargetExpressions());
+ } else {
+ throw new YarnRuntimeException(
+ "Encountered unexpected type of constraint target operator: "
+ + constraint.getOp());
+ }
+ return newConstraint;
+ }
+
+ @Override
+ public AbstractConstraint visit(CardinalityConstraint constraint) {
+ return new SingleConstraint(constraint.getScope(),
+ constraint.getMinCardinality(), constraint.getMaxCardinality(),
+ new TargetExpression(TargetExpression.TargetType.SELF));
+ }
+ }
+
+ /**
+ * Visits a {@link PlacementConstraint} tree and, whenever possible,
+ * substitutes each {@link SingleConstraint} with a {@link TargetConstraint}
+ * or a {@link CardinalityConstraint}. When such a substitution is not
+ * possible, we keep the original {@link SingleConstraint}.
+ */
+ public static class SpecializedConstraintTransformer
+ extends AbstractTransformer {
+
+ public SpecializedConstraintTransformer(PlacementConstraint constraint) {
+ super(constraint);
+ }
+
+ @Override
+ public AbstractConstraint visit(SingleConstraint constraint) {
+ AbstractConstraint transformedConstraint = constraint;
+ // Check if it is a cardinality constraint.
+ if (constraint.getTargetExpressions().size() == 1) {
+ TargetExpression targetExpr =
+ constraint.getTargetExpressions().iterator().next();
+ if (targetExpr.getTargetType() == TargetExpression.TargetType.SELF) {
+ transformedConstraint = new CardinalityConstraint(
+ constraint.getScope(), constraint.getMinCardinality(),
+ constraint.getMaxCardinality());
+ }
+ }
+ // Check if it is a target constraint.
+ if (constraint.getMinCardinality() == 1
+ && constraint.getMaxCardinality() == Integer.MAX_VALUE) {
+ transformedConstraint = new TargetConstraint(TargetOperator.IN,
+ constraint.getScope(), constraint.getTargetExpressions());
+ } else if (constraint.getMinCardinality() == 0
+ && constraint.getMaxCardinality() == 0) {
+ transformedConstraint = new TargetConstraint(TargetOperator.NOT_IN,
+ constraint.getScope(), constraint.getTargetExpressions());
+ }
+
+ return transformedConstraint;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
new file mode 100644
index 0000000..660dc02
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/resource/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * API related to resources.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.api.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
new file mode 100644
index 0000000..bd245e2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPlacementConstraintPBConversion.java
@@ -0,0 +1,195 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+import java.util.Iterator;
+
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintFromProtoConverter;
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintToProtoConverter;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.CompositePlacementConstraintProto.CompositeType;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SimplePlacementConstraintProto;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test class for {@link PlacementConstraintToProtoConverter} and
+ * {@link PlacementConstraintFromProtoConverter}.
+ */
+public class TestPlacementConstraintPBConversion {
+
+ @Test
+ public void testTargetConstraintProtoConverter() {
+ AbstractConstraint sConstraintExpr =
+ targetIn(NODE, allocationTag("hbase-m"));
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Convert to proto.
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(sConstraint);
+ PlacementConstraintProto protoConstraint = toProtoConverter.convert();
+
+ Assert.assertTrue(protoConstraint.hasSimpleConstraint());
+ Assert.assertFalse(protoConstraint.hasCompositeConstraint());
+ SimplePlacementConstraintProto sProto =
+ protoConstraint.getSimpleConstraint();
+ Assert.assertEquals(single.getScope(), sProto.getScope());
+ Assert.assertEquals(single.getMinCardinality(), sProto.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(), sProto.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions().size(),
+ sProto.getTargetExpressionsList().size());
+
+ // Convert from proto.
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(protoConstraint);
+ PlacementConstraint newConstraint = fromProtoConverter.convert();
+
+ AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
+ Assert.assertTrue(newConstraintExpr instanceof SingleConstraint);
+ SingleConstraint newSingle = (SingleConstraint) newConstraintExpr;
+ Assert.assertEquals(single.getScope(), newSingle.getScope());
+ Assert.assertEquals(single.getMinCardinality(),
+ newSingle.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(),
+ newSingle.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions(),
+ newSingle.getTargetExpressions());
+ }
+
+ @Test
+ public void testCardinalityConstraintProtoConverter() {
+ AbstractConstraint sConstraintExpr = cardinality(RACK, 3, 10);
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Convert to proto.
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(sConstraint);
+ PlacementConstraintProto protoConstraint = toProtoConverter.convert();
+
+ compareSimpleConstraintToProto(single, protoConstraint);
+
+ // Convert from proto.
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(protoConstraint);
+ PlacementConstraint newConstraint = fromProtoConverter.convert();
+
+ AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
+ Assert.assertTrue(newConstraintExpr instanceof SingleConstraint);
+ SingleConstraint newSingle = (SingleConstraint) newConstraintExpr;
+ compareSimpleConstraints(single, newSingle);
+ }
+
+ @Test
+ public void testCompositeConstraintProtoConverter() {
+ AbstractConstraint constraintExpr =
+ or(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
+ targetCardinality(RACK, 2, 10, allocationTag("zk")));
+ Assert.assertTrue(constraintExpr instanceof Or);
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+ Or orExpr = (Or) constraintExpr;
+
+ // Convert to proto.
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(constraint);
+ PlacementConstraintProto protoConstraint = toProtoConverter.convert();
+
+ Assert.assertFalse(protoConstraint.hasSimpleConstraint());
+ Assert.assertTrue(protoConstraint.hasCompositeConstraint());
+ CompositePlacementConstraintProto cProto =
+ protoConstraint.getCompositeConstraint();
+
+ Assert.assertEquals(CompositeType.OR, cProto.getCompositeType());
+ Assert.assertEquals(3, cProto.getChildConstraintsCount());
+ Assert.assertEquals(0, cProto.getTimedChildConstraintsCount());
+ Iterator<AbstractConstraint> orChildren = orExpr.getChildren().iterator();
+ Iterator<PlacementConstraintProto> orProtoChildren =
+ cProto.getChildConstraintsList().iterator();
+ while (orChildren.hasNext() && orProtoChildren.hasNext()) {
+ AbstractConstraint orChild = orChildren.next();
+ PlacementConstraintProto orProtoChild = orProtoChildren.next();
+ compareSimpleConstraintToProto((SingleConstraint) orChild, orProtoChild);
+ }
+
+ // Convert from proto.
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(protoConstraint);
+ PlacementConstraint newConstraint = fromProtoConverter.convert();
+
+ AbstractConstraint newConstraintExpr = newConstraint.getConstraintExpr();
+ Assert.assertTrue(newConstraintExpr instanceof Or);
+ Or newOrExpr = (Or) newConstraintExpr;
+ Assert.assertEquals(3, newOrExpr.getChildren().size());
+ orChildren = orExpr.getChildren().iterator();
+ Iterator<AbstractConstraint> newOrChildren =
+ newOrExpr.getChildren().iterator();
+ while (orChildren.hasNext() && newOrChildren.hasNext()) {
+ AbstractConstraint orChild = orChildren.next();
+ AbstractConstraint newOrChild = newOrChildren.next();
+ compareSimpleConstraints((SingleConstraint) orChild,
+ (SingleConstraint) newOrChild);
+ }
+ }
+
+ private void compareSimpleConstraintToProto(SingleConstraint constraint,
+ PlacementConstraintProto proto) {
+ Assert.assertTrue(proto.hasSimpleConstraint());
+ Assert.assertFalse(proto.hasCompositeConstraint());
+ SimplePlacementConstraintProto sProto = proto.getSimpleConstraint();
+ Assert.assertEquals(constraint.getScope(), sProto.getScope());
+ Assert.assertEquals(constraint.getMinCardinality(),
+ sProto.getMinCardinality());
+ Assert.assertEquals(constraint.getMaxCardinality(),
+ sProto.getMaxCardinality());
+ Assert.assertEquals(constraint.getTargetExpressions().size(),
+ sProto.getTargetExpressionsList().size());
+ }
+
+ private void compareSimpleConstraints(SingleConstraint single,
+ SingleConstraint newSingle) {
+ Assert.assertEquals(single.getScope(), newSingle.getScope());
+ Assert.assertEquals(single.getMinCardinality(),
+ newSingle.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(),
+ newSingle.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions(),
+ newSingle.getTargetExpressions());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90ac4f89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
new file mode 100644
index 0000000..1763735
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/resource/TestPlacementConstraintTransformations.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.resource;
+
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.NODE;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.RACK;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.cardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.maxCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.or;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetCardinality;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.targetIn;
+import static org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets.allocationTag;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.AbstractConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.CardinalityConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.Or;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.SingleConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint.TargetConstraint.TargetOperator;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SingleConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraintTransformations.SpecializedConstraintTransformer;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraints.PlacementTargets;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test class for {@link PlacementConstraintTransformations}.
+ */
+public class TestPlacementConstraintTransformations {
+
+ @Test
+ public void testTargetConstraint() {
+ AbstractConstraint sConstraintExpr =
+ targetIn(NODE, allocationTag("hbase-m"));
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Transform from SimpleConstraint to specialized TargetConstraint
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(sConstraint);
+ PlacementConstraint tConstraint = specTransformer.transform();
+
+ AbstractConstraint tConstraintExpr = tConstraint.getConstraintExpr();
+ Assert.assertTrue(tConstraintExpr instanceof TargetConstraint);
+
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ TargetConstraint target = (TargetConstraint) tConstraintExpr;
+ Assert.assertEquals(single.getScope(), target.getScope());
+ Assert.assertEquals(TargetOperator.IN, target.getOp());
+ Assert.assertEquals(single.getTargetExpressions(),
+ target.getTargetExpressions());
+
+ // Transform from specialized TargetConstraint to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(tConstraint);
+ sConstraint = singleTransformer.transform();
+
+ sConstraintExpr = sConstraint.getConstraintExpr();
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+
+ single = (SingleConstraint) sConstraintExpr;
+ Assert.assertEquals(target.getScope(), single.getScope());
+ Assert.assertEquals(1, single.getMinCardinality());
+ Assert.assertEquals(Integer.MAX_VALUE, single.getMaxCardinality());
+ Assert.assertEquals(single.getTargetExpressions(),
+ target.getTargetExpressions());
+ }
+
+ @Test
+ public void testCardinalityConstraint() {
+ AbstractConstraint sConstraintExpr = cardinality(RACK, 3, 10);
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+ PlacementConstraint sConstraint =
+ PlacementConstraints.build(sConstraintExpr);
+
+ // Transform from SimpleConstraint to specialized CardinalityConstraint
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(sConstraint);
+ PlacementConstraint cConstraint = specTransformer.transform();
+
+ AbstractConstraint cConstraintExpr = cConstraint.getConstraintExpr();
+ Assert.assertTrue(cConstraintExpr instanceof CardinalityConstraint);
+
+ SingleConstraint single = (SingleConstraint) sConstraintExpr;
+ CardinalityConstraint cardinality = (CardinalityConstraint) cConstraintExpr;
+ Assert.assertEquals(single.getScope(), cardinality.getScope());
+ Assert.assertEquals(single.getMinCardinality(),
+ cardinality.getMinCardinality());
+ Assert.assertEquals(single.getMaxCardinality(),
+ cardinality.getMaxCardinality());
+
+ // Transform from specialized CardinalityConstraint to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(cConstraint);
+ sConstraint = singleTransformer.transform();
+
+ sConstraintExpr = sConstraint.getConstraintExpr();
+ Assert.assertTrue(sConstraintExpr instanceof SingleConstraint);
+
+ single = (SingleConstraint) sConstraintExpr;
+ Assert.assertEquals(cardinality.getScope(), single.getScope());
+ Assert.assertEquals(cardinality.getMinCardinality(),
+ single.getMinCardinality());
+ Assert.assertEquals(cardinality.getMaxCardinality(),
+ single.getMaxCardinality());
+ Assert.assertEquals(new HashSet<>(Arrays.asList(PlacementTargets.self())),
+ single.getTargetExpressions());
+ }
+
+ @Test
+ public void testTargetCardinalityConstraint() {
+ AbstractConstraint constraintExpr =
+ targetCardinality(RACK, 3, 10, allocationTag("zk"));
+ Assert.assertTrue(constraintExpr instanceof SingleConstraint);
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+
+ // Apply transformation. Should be a no-op.
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(constraint);
+ PlacementConstraint newConstraint = specTransformer.transform();
+
+ // The constraint expression should be the same.
+ Assert.assertEquals(constraintExpr, newConstraint.getConstraintExpr());
+ }
+
+ @Test
+ public void testCompositeConstraint() {
+ AbstractConstraint constraintExpr =
+ or(targetIn(RACK, allocationTag("spark")), maxCardinality(NODE, 3),
+ targetCardinality(RACK, 2, 10, allocationTag("zk")));
+ Assert.assertTrue(constraintExpr instanceof Or);
+ PlacementConstraint constraint = PlacementConstraints.build(constraintExpr);
+ Or orExpr = (Or) constraintExpr;
+ for (AbstractConstraint child : orExpr.getChildren()) {
+ Assert.assertTrue(child instanceof SingleConstraint);
+ }
+
+ // Apply transformation. Should transform target and cardinality constraints
+ // included in the composite constraint to specialized ones.
+ SpecializedConstraintTransformer specTransformer =
+ new SpecializedConstraintTransformer(constraint);
+ PlacementConstraint specConstraint = specTransformer.transform();
+
+ Or specOrExpr = (Or) specConstraint.getConstraintExpr();
+ List<AbstractConstraint> specChildren = specOrExpr.getChildren();
+ Assert.assertEquals(3, specChildren.size());
+ Assert.assertTrue(specChildren.get(0) instanceof TargetConstraint);
+ Assert.assertTrue(specChildren.get(1) instanceof CardinalityConstraint);
+ Assert.assertTrue(specChildren.get(2) instanceof SingleConstraint);
+
+ // Transform from specialized TargetConstraint to SimpleConstraint
+ SingleConstraintTransformer singleTransformer =
+ new SingleConstraintTransformer(specConstraint);
+ PlacementConstraint simConstraint = singleTransformer.transform();
+ Assert.assertTrue(constraintExpr instanceof Or);
+ Or simOrExpr = (Or) specConstraint.getConstraintExpr();
+ for (AbstractConstraint child : simOrExpr.getChildren()) {
+ Assert.assertTrue(child instanceof SingleConstraint);
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/19] hadoop git commit: YARN-7448. [API] Add SchedulingRequest to
the AllocateRequest. (Panagiotis Garefalakis via asuresh)
Posted by as...@apache.org.
YARN-7448. [API] Add SchedulingRequest to the AllocateRequest. (Panagiotis Garefalakis via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9f0e942
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9f0e942
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9f0e942
Branch: refs/heads/YARN-6592
Commit: b9f0e942f92adc2a97b7f70151f57bdf2315bfe3
Parents: c24deea
Author: Arun Suresh <as...@apache.org>
Authored: Fri Nov 17 10:42:43 2017 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Nov 17 10:43:44 2017 -0800
----------------------------------------------------------------------
.../api/protocolrecords/AllocateRequest.java | 42 ++++++++++
.../hadoop/yarn/api/records/ResourceSizing.java | 27 +++++++
.../yarn/api/records/SchedulingRequest.java | 1 +
.../src/main/proto/yarn_service_protos.proto | 1 +
.../impl/pb/AllocateRequestPBImpl.java | 83 ++++++++++++++++++++
.../records/impl/pb/ResourceSizingPBImpl.java | 2 +-
.../impl/pb/SchedulingRequestPBImpl.java | 16 ++++
.../hadoop/yarn/api/TestPBImplRecords.java | 19 +++++
8 files changed, 190 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
index ae0891e..d8d2347 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateRequest.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.api.protocolrecords;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -28,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.util.Records;
@@ -212,6 +214,32 @@ public abstract class AllocateRequest {
public abstract void setUpdateRequests(
List<UpdateContainerRequest> updateRequests);
+ /**
+ * Get the list of Scheduling requests being sent by the
+ * <code>ApplicationMaster</code>.
+ * @return list of {@link SchedulingRequest} being sent by the
+ * <code>ApplicationMaster</code>.
+ */
+ @Public
+ @Unstable
+ public List<SchedulingRequest> getSchedulingRequests() {
+ return Collections.EMPTY_LIST;
+ }
+
+ /**
+ * Set the list of Scheduling requests to inform the
+ * <code>ResourceManager</code> about the application's resource requirements
+ * (potentially including allocation tags & placement constraints).
+ * @param schedulingRequests list of <code>SchedulingRequest</code> to update
+ * the <code>ResourceManager</code> about the application's resource
+ * requirements.
+ */
+ @Public
+ @Unstable
+ public void setSchedulingRequests(
+ List<SchedulingRequest> schedulingRequests) {
+ }
+
@Public
@Unstable
public static AllocateRequestBuilder newBuilder() {
@@ -314,6 +342,20 @@ public abstract class AllocateRequest {
}
/**
+ * Set the <code>schedulingRequests</code> of the request.
+ * @see AllocateRequest#setSchedulingRequests(List)
+ * @param schedulingRequests <code>SchedulingRequest</code> of the request
+ * @return {@link AllocateRequestBuilder}
+ */
+ @Public
+ @Unstable
+ public AllocateRequestBuilder schedulingRequests(
+ List<SchedulingRequest> schedulingRequests) {
+ allocateRequest.setSchedulingRequests(schedulingRequests);
+ return this;
+ }
+
+ /**
* Return generated {@link AllocateRequest} object.
* @return {@link AllocateRequest}
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
index d82be11..8cdc63f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
@@ -61,4 +61,31 @@ public abstract class ResourceSizing {
@Public
@Unstable
public abstract void setResources(Resource resources);
+
+ @Override
+ public int hashCode() {
+ int result = getResources().hashCode();
+ result = 31 * result + getNumAllocations();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if(obj == null || getClass() != obj.getClass()) {
+ return false;
+ }
+
+ ResourceSizing that = (ResourceSizing) obj;
+
+ if(getNumAllocations() != that.getNumAllocations()) {
+ return false;
+ }
+ if(!getResources().equals(that.getResources())) {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
index 47a0697..e32dd24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -49,6 +49,7 @@ public abstract class SchedulingRequest {
return SchedulingRequest.newBuilder()
.allocationRequestId(allocationRequestId).priority(priority)
.executionType(executionType).allocationTags(allocationTags)
+ .resourceSizing(resourceSizing)
.placementConstraintExpression(placementConstraintExpression).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 26bcb1d..5e7c213 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -91,6 +91,7 @@ message AllocateRequestProto {
optional int32 response_id = 4;
optional float progress = 5;
repeated UpdateContainerRequestProto update_requests = 7;
+ repeated SchedulingRequestProto scheduling_requests = 10;
}
message NMTokenProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
index 0f0f571..b460044 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateRequestPBImpl.java
@@ -29,14 +29,17 @@ import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.SchedulingRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.UpdateContainerRequestPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SchedulingRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.UpdateContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProtoOrBuilder;
@@ -53,6 +56,7 @@ public class AllocateRequestPBImpl extends AllocateRequest {
private List<ResourceRequest> ask = null;
private List<ContainerId> release = null;
private List<UpdateContainerRequest> updateRequests = null;
+ private List<SchedulingRequest> schedulingRequests = null;
private ResourceBlacklistRequest blacklistRequest = null;
public AllocateRequestPBImpl() {
@@ -101,6 +105,9 @@ public class AllocateRequestPBImpl extends AllocateRequest {
if (this.updateRequests != null) {
addUpdateRequestsToProto();
}
+ if (this.schedulingRequests != null) {
+ addSchedulingRequestsToProto();
+ }
if (this.blacklistRequest != null) {
builder.setBlacklistRequest(convertToProtoFormat(this.blacklistRequest));
}
@@ -178,6 +185,23 @@ public class AllocateRequestPBImpl extends AllocateRequest {
}
@Override
+ public List<SchedulingRequest> getSchedulingRequests() {
+ initSchedulingRequests();
+ return this.schedulingRequests;
+ }
+
+ @Override
+ public void setSchedulingRequests(
+ List<SchedulingRequest> schedulingRequests) {
+ if (schedulingRequests == null) {
+ return;
+ }
+ initSchedulingRequests();
+ this.schedulingRequests.clear();
+ this.schedulingRequests.addAll(schedulingRequests);
+ }
+
+ @Override
public ResourceBlacklistRequest getResourceBlacklistRequest() {
AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
if (this.blacklistRequest != null) {
@@ -261,6 +285,20 @@ public class AllocateRequestPBImpl extends AllocateRequest {
}
}
+ private void initSchedulingRequests() {
+ if (this.schedulingRequests != null) {
+ return;
+ }
+ AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List<SchedulingRequestProto> list =
+ p.getSchedulingRequestsList();
+ this.schedulingRequests = new ArrayList<>();
+
+ for (SchedulingRequestProto c : list) {
+ this.schedulingRequests.add(convertFromProtoFormat(c));
+ }
+ }
+
private void addUpdateRequestsToProto() {
maybeInitBuilder();
builder.clearUpdateRequests();
@@ -297,6 +335,41 @@ public class AllocateRequestPBImpl extends AllocateRequest {
builder.addAllUpdateRequests(iterable);
}
+ private void addSchedulingRequestsToProto() {
+ maybeInitBuilder();
+ builder.clearSchedulingRequests();
+ if (schedulingRequests == null) {
+ return;
+ }
+ Iterable<SchedulingRequestProto> iterable =
+ new Iterable<SchedulingRequestProto>() {
+ @Override
+ public Iterator<SchedulingRequestProto> iterator() {
+ return new Iterator<SchedulingRequestProto>() {
+
+ private Iterator<SchedulingRequest> iter =
+ schedulingRequests.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public SchedulingRequestProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+
+ }
+ };
+ builder.addAllSchedulingRequests(iterable);
+ }
@Override
public List<ContainerId> getReleaseList() {
initReleases();
@@ -377,6 +450,16 @@ public class AllocateRequestPBImpl extends AllocateRequest {
return ((UpdateContainerRequestPBImpl) t).getProto();
}
+ private SchedulingRequestPBImpl convertFromProtoFormat(
+ SchedulingRequestProto p) {
+ return new SchedulingRequestPBImpl(p);
+ }
+
+ private SchedulingRequestProto convertToProtoFormat(
+ SchedulingRequest t) {
+ return ((SchedulingRequestPBImpl) t).getProto();
+ }
+
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
return new ContainerIdPBImpl(p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
index 05bb3bd..f98e488 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
@@ -112,6 +112,6 @@ public class ResourceSizingPBImpl extends ResourceSizing {
}
private ResourceProto convertToProtoFormat(Resource r) {
- return ((ResourcePBImpl) r).getProto();
+ return ProtoUtils.convertToProtoFormat(r);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
index 7826b36..305856a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
@@ -263,4 +263,20 @@ public class SchedulingRequestPBImpl extends SchedulingRequest {
this.allocationTags = new HashSet<>();
this.allocationTags.addAll(p.getAllocationTagsList());
}
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9f0e942/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index c5585c2..a0b907d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -149,8 +149,10 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.api.records.Token;
@@ -189,7 +191,9 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ResourceBlacklistRequestPBImpl
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceOptionPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceSizingPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.SchedulingRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.StrictPreemptionContractPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
@@ -225,6 +229,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceBlacklistRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceOptionProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceSizingProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SchedulingRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
import org.apache.hadoop.yarn.proto.YarnProtos.StrictPreemptionContractProto;
import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
@@ -428,6 +434,8 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(QueueConfigurations.class);
generateByNewInstance(CollectorInfo.class);
generateByNewInstance(ResourceTypeInfo.class);
+ generateByNewInstance(ResourceSizing.class);
+ generateByNewInstance(SchedulingRequest.class);
}
@Test
@@ -907,6 +915,17 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
}
@Test
+ public void testResourceSizingPBImpl() throws Exception {
+ validatePBImplRecord(ResourceSizingPBImpl.class, ResourceSizingProto.class);
+ }
+
+ @Test
+ public void testSchedulingRequestPBImpl() throws Exception {
+ validatePBImplRecord(SchedulingRequestPBImpl.class,
+ SchedulingRequestProto.class);
+ }
+
+ @Test
public void testSerializedExceptionPBImpl() throws Exception {
validatePBImplRecord(SerializedExceptionPBImpl.class,
SerializedExceptionProto.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/19] hadoop git commit: YARN-6594. [API] Introduce
SchedulingRequest object. (Konstantinos Karanasos via wangda)
Posted by as...@apache.org.
YARN-6594. [API] Introduce SchedulingRequest object. (Konstantinos Karanasos via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff5a3204
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff5a3204
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff5a3204
Branch: refs/heads/YARN-6592
Commit: ff5a320465d499d42524c0f842f51192e7c5f0f5
Parents: 90ac4f8
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Oct 30 16:54:02 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Fri Nov 17 10:43:44 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/ResourceSizing.java | 64 +++++
.../yarn/api/records/SchedulingRequest.java | 205 ++++++++++++++
.../src/main/proto/yarn_protos.proto | 14 +
.../records/impl/pb/ResourceSizingPBImpl.java | 117 ++++++++
.../impl/pb/SchedulingRequestPBImpl.java | 266 +++++++++++++++++++
5 files changed, 666 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5a3204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
new file mode 100644
index 0000000..d82be11
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceSizing.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@code ResourceSizing} contains information for the size of a
+ * {@link SchedulingRequest}, such as the number of requested allocations and
+ * the resources for each allocation.
+ */
+@Public
+@Unstable
+public abstract class ResourceSizing {
+
+ @Public
+ @Unstable
+ public static ResourceSizing newInstance(Resource resources) {
+ return ResourceSizing.newInstance(1, resources);
+ }
+
+ @Public
+ @Unstable
+ public static ResourceSizing newInstance(int numAllocations, Resource resources) {
+ ResourceSizing resourceSizing = Records.newRecord(ResourceSizing.class);
+ resourceSizing.setNumAllocations(numAllocations);
+ resourceSizing.setResources(resources);
+ return resourceSizing;
+ }
+
+ @Public
+ @Unstable
+ public abstract int getNumAllocations();
+
+ @Public
+ @Unstable
+ public abstract void setNumAllocations(int numAllocations);
+
+ @Public
+ @Unstable
+ public abstract Resource getResources();
+
+ @Public
+ @Unstable
+ public abstract void setResources(Resource resources);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5a3204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
new file mode 100644
index 0000000..47a0697
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/SchedulingRequest.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * {@code SchedulingRequest} represents a request made by an application to the
+ * {@code ResourceManager} to obtain an allocation. It is similar to the
+ * {@link ResourceRequest}. However, it is more complete than the latter, as it
+ * allows applications to specify allocation tags (e.g., to express that an
+ * allocation belongs to {@code Spark} or is an {@code HBase-master}), as well
+ * as involved {@link PlacementConstraint}s (e.g., anti-affinity between Spark
+ * and HBase allocations).
+ *
+ * The size specification of the allocation is in {@code ResourceSizing}.
+ */
+@Public
+@Unstable
+public abstract class SchedulingRequest {
+
+ @Public
+ @Unstable
+ public static SchedulingRequest newInstance(long allocationRequestId,
+ Priority priority, ExecutionTypeRequest executionType,
+ Set<String> allocationTags, ResourceSizing resourceSizing,
+ PlacementConstraint placementConstraintExpression) {
+ return SchedulingRequest.newBuilder()
+ .allocationRequestId(allocationRequestId).priority(priority)
+ .executionType(executionType).allocationTags(allocationTags)
+ .placementConstraintExpression(placementConstraintExpression).build();
+ }
+
+ @Public
+ @Unstable
+ public static SchedulingRequestBuilder newBuilder() {
+ return new SchedulingRequestBuilder();
+ }
+
+ /**
+ * Class to construct instances of {@link SchedulingRequest} with specific
+ * options.
+ */
+ @Public
+ @Unstable
+ public static final class SchedulingRequestBuilder {
+ private SchedulingRequest schedulingRequest =
+ Records.newRecord(SchedulingRequest.class);
+
+ private SchedulingRequestBuilder() {
+ schedulingRequest.setAllocationRequestId(0);
+ schedulingRequest.setPriority(Priority.newInstance(0));
+ schedulingRequest.setExecutionType(ExecutionTypeRequest.newInstance());
+ }
+
+ /**
+ * Set the <code>allocationRequestId</code> of the request.
+ *
+ * @see SchedulingRequest#setAllocationRequestId(long)
+ * @param allocationRequestId <code>allocationRequestId</code> of the
+ * request
+ * @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ */
+ @Public
+ @Unstable
+ public SchedulingRequestBuilder allocationRequestId(
+ long allocationRequestId) {
+ schedulingRequest.setAllocationRequestId(allocationRequestId);
+ return this;
+ }
+
+ /**
+ * Set the <code>priority</code> of the request.
+ *
+ * @param priority <code>priority</code> of the request
+ * @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ * @see SchedulingRequest#setPriority(Priority)
+ */
+ @Public
+ @Unstable
+ public SchedulingRequestBuilder priority(Priority priority) {
+ schedulingRequest.setPriority(priority);
+ return this;
+ }
+
+ /**
+ * Set the <code>executionType</code> of the request.
+ *
+ * @see SchedulingRequest#setExecutionType(ExecutionTypeRequest)
+ * @param executionType <code>executionType</code> of the request
+ * @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ */
+ @Public
+ @Unstable
+ public SchedulingRequestBuilder executionType(
+ ExecutionTypeRequest executionType) {
+ schedulingRequest.setExecutionType(executionType);
+ return this;
+ }
+
+ /**
+ * Set the <code>allocationTags</code> of the request.
+ *
+ * @see SchedulingRequest#setAllocationTags(Set)
+ * @param allocationTags <code>allocationsTags</code> of the request
+ * @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ */
+ @Public
+ @Unstable
+ public SchedulingRequestBuilder allocationTags(Set<String> allocationTags) {
+ schedulingRequest.setAllocationTags(allocationTags);
+ return this;
+ }
+
+ /**
+ * Set the <code>executionType</code> of the request.
+ *
+ * @see SchedulingRequest#setResourceSizing(ResourceSizing)
+ * @param resourceSizing <code>resourceSizing</code> of the request
+ * @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ */
+ @Public
+ @Unstable
+ public SchedulingRequestBuilder resourceSizing(
+ ResourceSizing resourceSizing) {
+ schedulingRequest.setResourceSizing(resourceSizing);
+ return this;
+ }
+
+ /**
+ * Set the <code>placementConstraintExpression</code> of the request.
+ *
+ * @see SchedulingRequest#setPlacementConstraint(
+ * PlacementConstraint)
+ * @param placementConstraintExpression <code>placementConstraints</code> of
+ * the request
+ * @return {@link SchedulingRequest.SchedulingRequestBuilder}
+ */
+ @Public
+ @Unstable
+ public SchedulingRequestBuilder placementConstraintExpression(
+ PlacementConstraint placementConstraintExpression) {
+ schedulingRequest
+ .setPlacementConstraint(placementConstraintExpression);
+ return this;
+ }
+
+ /**
+ * Return generated {@link SchedulingRequest} object.
+ *
+ * @return {@link SchedulingRequest}
+ */
+ @Public
+ @Unstable
+ public SchedulingRequest build() {
+ return schedulingRequest;
+ }
+ }
+
+ public abstract long getAllocationRequestId();
+
+ public abstract void setAllocationRequestId(long allocationRequestId);
+
+ public abstract Priority getPriority();
+
+ public abstract void setPriority(Priority priority);
+
+ public abstract ExecutionTypeRequest getExecutionType();
+
+ public abstract void setExecutionType(ExecutionTypeRequest executionType);
+
+ public abstract Set<String> getAllocationTags();
+
+ public abstract void setAllocationTags(Set<String> allocationTags);
+
+ public abstract ResourceSizing getResourceSizing();
+
+ public abstract void setResourceSizing(ResourceSizing resourceSizing);
+
+ public abstract PlacementConstraint getPlacementConstraint();
+
+ public abstract void setPlacementConstraint(
+ PlacementConstraint placementConstraint);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5a3204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3739d94..09feb1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -397,6 +397,20 @@ message ExecutionTypeRequestProto {
optional bool enforce_execution_type = 2 [default = false];
}
+message SchedulingRequestProto {
+ optional int64 allocationRequestId = 1 [default = 0];
+ optional PriorityProto priority = 2;
+ optional ExecutionTypeRequestProto executionType = 3;
+ repeated string allocationTags = 4;
+ optional ResourceSizingProto resourceSizing = 5;
+ optional PlacementConstraintProto placementConstraint = 6;
+}
+
+message ResourceSizingProto {
+ optional int32 numAllocations = 1;
+ optional ResourceProto resources = 2;
+}
+
enum AMCommandProto {
AM_RESYNC = 1;
AM_SHUTDOWN = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5a3204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
new file mode 100644
index 0000000..05bb3bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceSizingPBImpl.java
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceSizingProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceSizingProtoOrBuilder;
+
+@Private
+@Unstable
+public class ResourceSizingPBImpl extends ResourceSizing {
+ ResourceSizingProto proto = ResourceSizingProto.getDefaultInstance();
+ ResourceSizingProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private Resource resources = null;
+
+ public ResourceSizingPBImpl() {
+ builder = ResourceSizingProto.newBuilder();
+ }
+
+ public ResourceSizingPBImpl(ResourceSizingProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ResourceSizingProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.resources != null) {
+ builder.setResources(convertToProtoFormat(this.resources));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceSizingProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int getNumAllocations() {
+ ResourceSizingProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getNumAllocations());
+ }
+
+ @Override
+ public void setNumAllocations(int numAllocations) {
+ maybeInitBuilder();
+ builder.setNumAllocations(numAllocations);
+ }
+
+ @Override
+ public Resource getResources() {
+ ResourceSizingProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.resources != null) {
+ return this.resources;
+ }
+ if (!p.hasResources()) {
+ return null;
+ }
+ this.resources = convertFromProtoFormat(p.getResources());
+ return this.resources;
+ }
+
+ @Override
+ public void setResources(Resource resources) {
+ maybeInitBuilder();
+ if (resources == null) {
+ builder.clearResources();
+ }
+ this.resources = resources;
+ }
+
+ private ResourcePBImpl convertFromProtoFormat(ResourceProto r) {
+ return new ResourcePBImpl(r);
+ }
+
+ private ResourceProto convertToProtoFormat(Resource r) {
+ return ((ResourcePBImpl) r).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff5a3204/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
new file mode 100644
index 0000000..7826b36
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SchedulingRequestPBImpl.java
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintFromProtoConverter;
+import org.apache.hadoop.yarn.api.pb.PlacementConstraintToProtoConverter;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ResourceSizing;
+import org.apache.hadoop.yarn.api.records.SchedulingRequest;
+import org.apache.hadoop.yarn.api.resource.PlacementConstraint;
+import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PlacementConstraintProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceSizingProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SchedulingRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.SchedulingRequestProtoOrBuilder;
+
+@Private
+@Unstable
+public class SchedulingRequestPBImpl extends SchedulingRequest {
+ SchedulingRequestProto proto = SchedulingRequestProto.getDefaultInstance();
+ SchedulingRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private Priority priority = null;
+ private ExecutionTypeRequest executionType = null;
+ private Set<String> allocationTags = null;
+ private ResourceSizing resourceSizing = null;
+ private PlacementConstraint placementConstraint = null;
+
+ public SchedulingRequestPBImpl() {
+ builder = SchedulingRequestProto.newBuilder();
+ }
+
+ public SchedulingRequestPBImpl(SchedulingRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public SchedulingRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.priority != null) {
+ builder.setPriority(convertToProtoFormat(this.priority));
+ }
+ if (this.executionType != null) {
+ builder.setExecutionType(convertToProtoFormat(this.executionType));
+ }
+ if (this.allocationTags != null) {
+ builder.clearAllocationTags();
+ builder.addAllAllocationTags(this.allocationTags);
+ }
+ if (this.resourceSizing != null) {
+ builder.setResourceSizing(convertToProtoFormat(this.resourceSizing));
+ }
+ if (this.placementConstraint != null) {
+ builder.setPlacementConstraint(
+ convertToProtoFormat(this.placementConstraint));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = SchedulingRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public long getAllocationRequestId() {
+ SchedulingRequestProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getAllocationRequestId());
+ }
+
+ @Override
+ public void setAllocationRequestId(long allocationRequestId) {
+ maybeInitBuilder();
+ builder.setAllocationRequestId(allocationRequestId);
+ }
+
+ @Override
+ public Priority getPriority() {
+ SchedulingRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.priority != null) {
+ return this.priority;
+ }
+ if (!p.hasPriority()) {
+ return null;
+ }
+ this.priority = convertFromProtoFormat(p.getPriority());
+ return this.priority;
+ }
+
+ @Override
+ public void setPriority(Priority priority) {
+ maybeInitBuilder();
+ if (priority == null) {
+ builder.clearPriority();
+ }
+ this.priority = priority;
+ }
+
+ @Override
+ public ExecutionTypeRequest getExecutionType() {
+ SchedulingRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.executionType != null) {
+ return this.executionType;
+ }
+ if (!p.hasExecutionType()) {
+ return null;
+ }
+ this.executionType = convertFromProtoFormat(p.getExecutionType());
+ return this.executionType;
+ }
+
+ @Override
+ public void setExecutionType(ExecutionTypeRequest executionType) {
+ maybeInitBuilder();
+ if (executionType == null) {
+ builder.clearExecutionType();
+ }
+ this.executionType = executionType;
+ }
+
+ @Override
+ public Set<String> getAllocationTags() {
+ initAllocationTags();
+ return this.allocationTags;
+ }
+
+ @Override
+ public void setAllocationTags(Set<String> allocationTags) {
+ maybeInitBuilder();
+ builder.clearAllocationTags();
+ this.allocationTags = allocationTags;
+ }
+
+ @Override
+ public ResourceSizing getResourceSizing() {
+ SchedulingRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.resourceSizing != null) {
+ return this.resourceSizing;
+ }
+ if (!p.hasResourceSizing()) {
+ return null;
+ }
+ this.resourceSizing = convertFromProtoFormat(p.getResourceSizing());
+ return this.resourceSizing;
+ }
+
+ @Override
+ public void setResourceSizing(ResourceSizing resourceSizing) {
+ maybeInitBuilder();
+ if (resourceSizing == null) {
+ builder.clearResourceSizing();
+ }
+ this.resourceSizing = resourceSizing;
+ }
+
+ @Override
+ public PlacementConstraint getPlacementConstraint() {
+ SchedulingRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.placementConstraint != null) {
+ return this.placementConstraint;
+ }
+ if (!p.hasPlacementConstraint()) {
+ return null;
+ }
+ this.placementConstraint =
+ convertFromProtoFormat(p.getPlacementConstraint());
+ return this.placementConstraint;
+ }
+
+ @Override
+ public void setPlacementConstraint(PlacementConstraint placementConstraint) {
+ maybeInitBuilder();
+ if (placementConstraint == null) {
+ builder.clearPlacementConstraint();
+ }
+ this.placementConstraint = placementConstraint;
+ }
+
+ private PriorityPBImpl convertFromProtoFormat(PriorityProto p) {
+ return new PriorityPBImpl(p);
+ }
+
+ private PriorityProto convertToProtoFormat(Priority p) {
+ return ((PriorityPBImpl) p).getProto();
+ }
+
+ private ExecutionTypeRequestPBImpl convertFromProtoFormat(
+ ExecutionTypeRequestProto p) {
+ return new ExecutionTypeRequestPBImpl(p);
+ }
+
+ private ExecutionTypeRequestProto convertToProtoFormat(
+ ExecutionTypeRequest p) {
+ return ((ExecutionTypeRequestPBImpl) p).getProto();
+ }
+
+ private ResourceSizingPBImpl convertFromProtoFormat(ResourceSizingProto p) {
+ return new ResourceSizingPBImpl(p);
+ }
+
+ private ResourceSizingProto convertToProtoFormat(ResourceSizing p) {
+ return ((ResourceSizingPBImpl) p).getProto();
+ }
+
+ private PlacementConstraint convertFromProtoFormat(
+ PlacementConstraintProto c) {
+ PlacementConstraintFromProtoConverter fromProtoConverter =
+ new PlacementConstraintFromProtoConverter(c);
+ return fromProtoConverter.convert();
+ }
+
+ private PlacementConstraintProto convertToProtoFormat(PlacementConstraint c) {
+ PlacementConstraintToProtoConverter toProtoConverter =
+ new PlacementConstraintToProtoConverter(c);
+ return toProtoConverter.convert();
+ }
+
+ private void initAllocationTags() {
+ if (this.allocationTags != null) {
+ return;
+ }
+ SchedulingRequestProtoOrBuilder p = viaProto ? proto : builder;
+ this.allocationTags = new HashSet<>();
+ this.allocationTags.addAll(p.getAllocationTagsList());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/19] hadoop git commit: YARN-7218. Decouple YARN Services REST API
namespace from RM. (Contributed by Eric Yang)
Posted by as...@apache.org.
YARN-7218. Decouple YARN Services REST API namespace from RM. (Contributed by Eric Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0940e4f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0940e4f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0940e4f6
Branch: refs/heads/YARN-6592
Commit: 0940e4f692441f16e742666ac925f71a083eab27
Parents: 5f0b238
Author: Eric Yang <ey...@apache.org>
Authored: Fri Nov 17 12:28:12 2017 -0500
Committer: Eric Yang <ey...@apache.org>
Committed: Fri Nov 17 12:28:12 2017 -0500
----------------------------------------------------------------------
.../hadoop/yarn/service/TestApiServer.java | 4 +-
.../yarn/service/api/records/Component.java | 51 +++++++++++++-------
.../service/api/records/ReadinessCheck.java | 14 +++++-
.../yarn/service/api/records/Service.java | 15 +++---
.../yarn/service/conf/RestApiConstants.java | 2 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 10 +++-
.../server/resourcemanager/ResourceManager.java | 14 +++++-
.../server/resourcemanager/webapp/RMWebApp.java | 15 ------
8 files changed, 77 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
index 2b22474..896b2f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/test/java/org/apache/hadoop/yarn/service/TestApiServer.java
@@ -62,8 +62,8 @@ public class TestApiServer {
this.apiServer.getClass().isAnnotationPresent(Path.class));
final Path path = this.apiServer.getClass()
.getAnnotation(Path.class);
- assertEquals("The path has /ws/v1 annotation", path.value(),
- "/ws/v1");
+ assertEquals("The path has /v1 annotation", path.value(),
+ "/v1");
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
index fe9c043..ce0e0cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Component.java
@@ -26,6 +26,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@@ -46,22 +48,53 @@ import org.apache.hadoop.classification.InterfaceStability;
@ApiModel(description = "One or more components of the service. If the service is HBase say, then the component can be a simple role like master or regionserver. If the service is a complex business webapp then a component can be other services say Kafka or Storm. Thereby it opens up the support for complex and nested services.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
@JsonInclude(JsonInclude.Include.NON_NULL)
public class Component implements Serializable {
private static final long serialVersionUID = -8430058381509087805L;
+ @JsonProperty("name")
private String name = null;
+
+ @JsonProperty("dependencies")
private List<String> dependencies = new ArrayList<String>();
+
+ @JsonProperty("readiness_check")
+ @XmlElement(name = "readiness_check")
private ReadinessCheck readinessCheck = null;
+
+ @JsonProperty("artifact")
private Artifact artifact = null;
+
+ @JsonProperty("launch_command")
+ @XmlElement(name = "launch_command")
private String launchCommand = null;
+
+ @JsonProperty("resource")
private Resource resource = null;
+
+ @JsonProperty("number_of_containers")
+ @XmlElement(name = "number_of_containers")
private Long numberOfContainers = null;
+
+ @JsonProperty("run_privileged_container")
+ @XmlElement(name = "run_privileged_container")
private Boolean runPrivilegedContainer = false;
+
+ @JsonProperty("placement_policy")
+ @XmlElement(name = "placement_policy")
private PlacementPolicy placementPolicy = null;
+
+ @JsonProperty("state")
private ComponentState state = ComponentState.FLEXING;
+
+ @JsonProperty("configuration")
private Configuration configuration = new Configuration();
+
+ @JsonProperty("quicklinks")
private List<String> quicklinks = new ArrayList<String>();
+
+ @JsonProperty("containers")
private List<Container> containers =
Collections.synchronizedList(new ArrayList<Container>());
@@ -74,7 +107,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", required = true, value = "Name of the service component (mandatory).")
- @JsonProperty("name")
public String getName() {
return name;
}
@@ -95,7 +127,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "An array of service components which should be in READY state (as defined by readiness check), before this component can be started. The dependencies across all components of an service should be represented as a DAG.")
- @JsonProperty("dependencies")
public List<String> getDependencies() {
return dependencies;
}
@@ -113,12 +144,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Readiness check for this component.")
- @JsonProperty("readiness_check")
public ReadinessCheck getReadinessCheck() {
return readinessCheck;
}
- @XmlElement(name = "readiness_check")
public void setReadinessCheck(ReadinessCheck readinessCheck) {
this.readinessCheck = readinessCheck;
}
@@ -133,7 +162,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Artifact of the component (optional). If not specified, the service level global artifact takes effect.")
- @JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
}
@@ -153,12 +181,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "The custom launch command of this component (optional). When specified at the component level, it overrides the value specified at the global level (if any).")
- @JsonProperty("launch_command")
public String getLaunchCommand() {
return launchCommand;
}
- @XmlElement(name = "launch_command")
public void setLaunchCommand(String launchCommand) {
this.launchCommand = launchCommand;
}
@@ -173,7 +199,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Resource of this component (optional). If not specified, the service level global resource takes effect.")
- @JsonProperty("resource")
public Resource getResource() {
return resource;
}
@@ -192,18 +217,15 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Number of containers for this component (optional). If not specified, the service level global number_of_containers takes effect.")
- @JsonProperty("number_of_containers")
public Long getNumberOfContainers() {
return numberOfContainers;
}
- @XmlElement(name = "number_of_containers")
public void setNumberOfContainers(Long numberOfContainers) {
this.numberOfContainers = numberOfContainers;
}
@ApiModelProperty(example = "null", value = "Containers of a started component. Specifying a value for this attribute for the POST payload raises a validation error. This blob is available only in the GET response of a started service.")
- @JsonProperty("containers")
public List<Container> getContainers() {
return containers;
}
@@ -237,12 +259,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Run all containers of this component in privileged mode (YARN-4262).")
- @JsonProperty("run_privileged_container")
public Boolean getRunPrivilegedContainer() {
return runPrivilegedContainer;
}
- @XmlElement(name = "run_privileged_container")
public void setRunPrivilegedContainer(Boolean runPrivilegedContainer) {
this.runPrivilegedContainer = runPrivilegedContainer;
}
@@ -259,12 +279,10 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies for all containers of this component (optional). If not specified, the service level placement_policy takes effect. Refer to the description at the global level for more details.")
- @JsonProperty("placement_policy")
public PlacementPolicy getPlacementPolicy() {
return placementPolicy;
}
- @XmlElement(name = "placement_policy")
public void setPlacementPolicy(PlacementPolicy placementPolicy) {
this.placementPolicy = placementPolicy;
}
@@ -278,7 +296,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "Config properties for this component.")
- @JsonProperty("configuration")
public Configuration getConfiguration() {
return configuration;
}
@@ -297,7 +314,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "A list of quicklink keys defined at the service level, and to be resolved by this component.")
- @JsonProperty("quicklinks")
public List<String> getQuicklinks() {
return quicklinks;
}
@@ -312,7 +328,6 @@ public class Component implements Serializable {
}
@ApiModelProperty(example = "null", value = "State of the component.")
- @JsonProperty("state")
public ComponentState getState() {
return state;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
index 0a3713c..2bcf68b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ReadinessCheck.java
@@ -25,7 +25,11 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlType;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -43,6 +47,8 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
@ApiModel(description = "A custom command or a pluggable helper container to determine the readiness of a container of a component. Readiness for every service is different. Hence the need for a simple interface, with scope to support advanced usecases.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
public class ReadinessCheck implements Serializable {
private static final long serialVersionUID = -3836839816887186801L;
@@ -68,8 +74,14 @@ public class ReadinessCheck implements Serializable {
}
}
+ @JsonProperty("type")
+ @XmlElement(name = "type")
private TypeEnum type = null;
+ @JsonProperty("properties")
+ @XmlElement(name = "properties")
private Map<String, String> properties = new HashMap<String, String>();
+ @JsonProperty("artifact")
+ @XmlElement(name = "artifact")
private Artifact artifact = null;
/**
@@ -82,7 +94,6 @@ public class ReadinessCheck implements Serializable {
}
@ApiModelProperty(example = "null", value = "E.g. HTTP (YARN will perform a simple REST call at a regular interval and expect a 204 No content).")
- @JsonProperty("type")
public TypeEnum getType() {
return type;
}
@@ -129,7 +140,6 @@ public class ReadinessCheck implements Serializable {
}
@ApiModelProperty(example = "null", value = "Artifact of the pluggable readiness check helper container (optional). If specified, this helper container typically hosts the http uri and encapsulates the complex scripts required to perform actual container readiness check. At the end it is expected to respond a 204 No content just like the simplified use case. This pluggable framework benefits service owners who can run services without any packaging modifications. Note, artifacts of type docker only is supported for now.")
- @JsonProperty("artifact")
public Artifact getArtifact() {
return artifact;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
index 77a2610..8045822 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/Service.java
@@ -26,6 +26,8 @@ import io.swagger.annotations.ApiModelProperty;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.ArrayList;
@@ -43,6 +45,7 @@ import java.util.Objects;
@ApiModel(description = "An Service resource has the following attributes.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonPropertyOrder({ "name", "state", "resource", "number_of_containers",
"lifetime", "containers" })
@@ -53,9 +56,15 @@ public class Service extends BaseResource {
private String id = null;
private Artifact artifact = null;
private Resource resource = null;
+ @JsonProperty("launch_time")
+ @XmlElement(name = "launch_time")
private Date launchTime = null;
+ @JsonProperty("number_of_running_containers")
+ @XmlElement(name = "number_of_running_containers")
private Long numberOfRunningContainers = null;
private Long lifetime = null;
+ @JsonProperty("placement_policy")
+ @XmlElement(name = "placement_policy")
private PlacementPolicy placementPolicy = null;
private List<Component> components = new ArrayList<>();
private Configuration configuration = new Configuration();
@@ -148,12 +157,10 @@ public class Service extends BaseResource {
}
@ApiModelProperty(example = "null", value = "The time when the service was created, e.g. 2016-03-16T01:01:49.000Z.")
- @JsonProperty("launch_time")
public Date getLaunchTime() {
return launchTime == null ? null : (Date) launchTime.clone();
}
- @XmlElement(name = "launch_time")
public void setLaunchTime(Date launchTime) {
this.launchTime = launchTime == null ? null : (Date) launchTime.clone();
}
@@ -171,12 +178,10 @@ public class Service extends BaseResource {
}
@ApiModelProperty(example = "null", value = "In get response this provides the total number of running containers for this service (across all components) at the time of request. Note, a subsequent request can return a different number as and when more containers get allocated until it reaches the total number of containers or if a flex request has been made between the two requests.")
- @JsonProperty("number_of_running_containers")
public Long getNumberOfRunningContainers() {
return numberOfRunningContainers;
}
- @XmlElement(name = "number_of_running_containers")
public void setNumberOfRunningContainers(Long numberOfRunningContainers) {
this.numberOfRunningContainers = numberOfRunningContainers;
}
@@ -215,12 +220,10 @@ public class Service extends BaseResource {
}
@ApiModelProperty(example = "null", value = "Advanced scheduling and placement policies (optional). If not specified, it defaults to the default placement policy of the service owner. The design of placement policies are in the works. It is not very clear at this point, how policies in conjunction with labels be exposed to service owners. This is a placeholder for now. The advanced structure of this attribute will be determined by YARN-4902.")
- @JsonProperty("placement_policy")
public PlacementPolicy getPlacementPolicy() {
return placementPolicy;
}
- @XmlElement(name = "placement_policy")
public void setPlacementPolicy(PlacementPolicy placementPolicy) {
this.placementPolicy = placementPolicy;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
index 35e1980..243fc52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/RestApiConstants.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.service.conf;
public interface RestApiConstants {
// Rest endpoints
- String CONTEXT_ROOT = "/ws/v1";
+ String CONTEXT_ROOT = "/v1";
String VERSION = "/services/version";
String SERVICE_ROOT_PATH = "/services";
String SERVICE_PATH = "/services/{service_name}";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index 4f1cacf..d3ad53e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -394,16 +394,22 @@ public class WebApps {
}
public WebApp start(WebApp webapp) {
- return start(webapp, null);
+ return start(webapp, null, null);
}
- public WebApp start(WebApp webapp, WebAppContext ui2Context) {
+ public WebApp start(WebApp webapp, WebAppContext ui2Context,
+ Map<String, String> services) {
WebApp webApp = build(webapp);
HttpServer2 httpServer = webApp.httpServer();
if (ui2Context != null) {
addFiltersForNewContext(ui2Context);
httpServer.addHandlerAtFront(ui2Context);
}
+ if (services!=null) {
+ String packageName = services.get("PackageName");
+ String pathSpec = services.get("PathSpec");
+ httpServer.addJerseyResourcePackage(packageName, pathSpec);
+ }
try {
httpServer.start();
LOG.info("Web app " + name + " started at "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 727bc52..6f8a0a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -126,7 +126,9 @@ import java.nio.charset.Charset;
import java.security.PrivilegedExceptionAction;
import java.security.SecureRandom;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
/**
@@ -1062,7 +1064,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected void startWepApp() {
-
+ Map<String, String> serviceConfig = null;
Configuration conf = getConfig();
RMWebAppUtil.setupSecurityAndFilters(conf,
@@ -1128,7 +1130,15 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
- webApp = builder.start(new RMWebApp(this), uiWebAppContext);
+ if (getConfig().getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE,
+ false)) {
+ serviceConfig = new HashMap<String, String>();
+ String apiPackages = "org.apache.hadoop.yarn.service.webapp;" +
+ "org.apache.hadoop.yarn.webapp";
+ serviceConfig.put("PackageName", apiPackages);
+ serviceConfig.put("PathSpec", "/app/*");
+ }
+ webApp = builder.start(new RMWebApp(this), uiWebAppContext, serviceConfig);
}
private String getWebAppsPath(String appName) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0940e4f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index bee9354..b8d4390 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -44,8 +44,6 @@ public class RMWebApp extends WebApp implements YarnWebParams {
LogFactory.getLog(RMWebApp.class.getName());
private final ResourceManager rm;
private boolean standby = false;
- private final static String APISERVER =
- "org.apache.hadoop.yarn.service.webapp.ApiServer";
public RMWebApp(ResourceManager rm) {
this.rm = rm;
@@ -59,19 +57,6 @@ public class RMWebApp extends WebApp implements YarnWebParams {
bind(RMWebApp.class).toInstance(this);
if (rm != null) {
- boolean enableServiceApi = rm.getConfig()
- .getBoolean(YarnConfiguration.YARN_API_SERVICES_ENABLE, false);
- if (enableServiceApi) {
- try {
- // Use reflection here to load ApiServer class,
- // this is done to avoid creating cyclic dependency
- // between maven projects.
- Class<?> apiServer = Class.forName(APISERVER);
- bind(apiServer);
- } catch (ClassNotFoundException e) {
- LOG.warn("ApiServer REST API is not activated.");
- }
- }
bind(ResourceManager.class).toInstance(rm);
}
route("/", RmController.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/19] hadoop git commit: YARN-7492. Set up SASS for new YARN UI
styling. Contributed by Vasudevan Skm.
Posted by as...@apache.org.
YARN-7492. Set up SASS for new YARN UI styling. Contributed by Vasudevan Skm.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09a13426
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09a13426
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09a13426
Branch: refs/heads/YARN-6592
Commit: 09a13426086aed9d1bc63a844858dcac560763a6
Parents: 675e9a8
Author: Sunil G <su...@apache.org>
Authored: Thu Nov 16 13:44:21 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Nov 16 13:44:21 2017 +0530
----------------------------------------------------------------------
.gitignore | 3 +
.../main/webapp/app/components/nodes-heatmap.js | 6 +-
...er-app-memusage-by-nodes-stacked-barchart.js | 2 +-
.../main/webapp/app/components/tree-selector.js | 6 +-
.../src/main/webapp/app/styles/app.css | 717 ------------------
.../src/main/webapp/app/styles/app.scss | 723 +++++++++++++++++++
.../src/main/webapp/app/styles/colors.scss | 37 +
.../src/main/webapp/app/styles/variables.scss | 40 +
.../src/main/webapp/app/utils/color-utils.js | 8 +-
.../hadoop-yarn-ui/src/main/webapp/package.json | 3 +-
.../hadoop-yarn-ui/src/main/webapp/yarn.lock | 674 ++++++++++++++++-
11 files changed, 1477 insertions(+), 742 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 70c1f23..440708a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@
*.sdf
*.suo
*.vcxproj.user
+*.patch
.idea
.svn
.classpath
@@ -45,3 +46,5 @@ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/tmp
yarnregistry.pdf
patchprocess/
.history/
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
index 1ea655b..7802d42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/nodes-heatmap.js
@@ -122,8 +122,8 @@ export default BaseChartComponent.extend({
var xOffset = layout.margin;
var yOffset = layout.margin * 3;
- var gradientStartColor = "#2ca02c";
- var gradientEndColor = "#ffb014";
+ var gradientStartColor = "#60cea5";
+ var gradientEndColor = "#ffbc0b";
var colorFunc = d3.interpolateRgb(d3.rgb(gradientStartColor), d3.rgb(gradientEndColor));
@@ -138,7 +138,7 @@ export default BaseChartComponent.extend({
var rect = g.append("rect")
.attr("x", sampleXOffset)
.attr("y", sampleYOffset)
- .attr("fill", this.selectedCategory === i ? "#2c7bb6" : colorFunc(ratio))
+ .attr("fill", this.selectedCategory === i ? "#26bbf0" : colorFunc(ratio))
.attr("width", this.SAMPLE_CELL_WIDTH)
.attr("height", this.SAMPLE_HEIGHT)
.attr("class", "hyperlink");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/per-app-memusage-by-nodes-stacked-barchart.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/per-app-memusage-by-nodes-stacked-barchart.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/per-app-memusage-by-nodes-stacked-barchart.js
index 65cbaf5..c01fe36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/per-app-memusage-by-nodes-stacked-barchart.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/per-app-memusage-by-nodes-stacked-barchart.js
@@ -74,7 +74,7 @@ export default StackedBarchart.extend({
didInsertElement: function() {
this.initChart(true);
- this.colors = ["Orange", "Grey", "LimeGreen"];
+ this.colors = ["lightsalmon", "Grey", "mediumaquamarine"];
var containers = this.get("rmContainers");
var nodes = this.get("nodes");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
index 4a0e6aa..7a9d53b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
@@ -156,11 +156,11 @@ export default Ember.Component.extend({
maxCap = maxCap === undefined ? 100 : maxCap;
var usedCap = d.queueData.get(this.used) / maxCap * 100.0;
if (usedCap <= 60.0) {
- return "LimeGreen";
+ return "mediumaquamarine";
} else if (usedCap <= 100.0) {
- return "DarkOrange";
+ return "coral";
} else {
- return "LightCoral";
+ return "salmon";
}
}.bind(this));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css
deleted file mode 100644
index 04c484b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.css
+++ /dev/null
@@ -1,717 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-body, html, body > .ember-view {
- height: 100%;
- overflow: visible;
- background: #f6f6f8;
- font-family: "Open Sans","Helvetica Neue", sans-serif!important;
-}
-body, html {
- min-width: 1024px;
-}
-
-/*
- Over all style
- */
-text {
- font-size: 14px;
-}
-
-text.small {
- font-size: 8px;
-}
-
-html, body
-{
- margin: 0px;
- padding: 0px;
- height: 100%;
- width: 100%;
-}
-
-
-/*
- queue's style (left banner of queues)
- */
-
-text.queue {
- font-family : sans-serif;
- font-size : 15px;
- fill : gray;
-}
-
-text.heatmap-cell {
- font-size: 14px;
- text-anchor: middle;
- fill: Azure;
- text-align: center;
-}
-
-.hyperlink {
- cursor: pointer;
- -webkit-user-select: none;
- -moz-user-select: none;
- -ms-user-select: none;
- user-select: none;
-}
-
-.heatmap-clear {
- fill: #337ab7;
-}
-
-text.heatmap-cell-notselected {
- font: 14px sans-serif;
- font-weight: bold;
- text-anchor: middle;
- fill: Silver;
- text-align: center;
-
-}
-
-text.heatmap-rack {
- font-size: 18px;
- font-weight: 400;
- fill: #4b5052;
-}
-
-path.queue {
- stroke: "red";
- fill: none;
-}
-
-/*
- background style
- */
-line.grid {
- stroke: WhiteSmoke;
-}
-
-line.chart {
- stroke: Gray;
-}
-
-/*
- charts styles
- */
-text.chart-title {
- font-size: 30px;
- text-anchor: middle;
- fill: #4b5052;
-}
-
-text.donut-highlight-text, text.donut-highlight-sub {
- font-size: 15px;
- font-family: sans-serif;
- text-anchor: middle;
- fill: Gray;
- vertical-align: middle;
-}
-
-text.donut-highlight-sub {
- font-size: 23px;
- margin-top: 10px;
-}
-
-rect.chart-frame {
- fill: none;
-}
-
-text.bar-chart-text {
- font-size: 8px;
- font-family: sans-serif;
- vertical-align: middle;
- fill: Gray;;
-}
-
-div.tooltip {
- position: absolute;
- text-align: center;
- padding: 10px;
- font-size: 16px;
- background: black;
- color: white;
- border: 0px;
- border-radius: 8px;
- pointer-events: none;
-}
-
-/*
- * Loading icon styles
- */
-.loading-mask > img {
- display: block;
- margin: 0 auto;
-}
-
-/*
- * Data table
- */
-
-table.dataTable thead .sorting {
- background-image: url("/assets/images/datatables/sort_both.png");
-}
-table.dataTable thead .sorting_asc {
- background-image: url("/assets/images/datatables/sort_asc.png");
-}
-table.dataTable thead .sorting_desc {
- background-image: url("/assets/images/datatables/sort_desc.png");
-}
-table.dataTable thead .sorting_asc_disabled {
- background-image: url("/assets/images/datatables/sort_asc_disabled.png");
-}
-table.dataTable thead .sorting_desc_disabled {
- background-image: url("/assets/images/datatables/sort_desc_disabled.png");
-}
-
-.add-ellipsis {
- overflow: hidden;
- text-overflow: ellipsis;
-}
-
-.breadcrumb {
- padding-bottom: 3px;
- background-color: #f6f6f8;
-}
-
-.navbar-default .navbar-nav > li > a {
- color: #337ab7;
-}
-
-/*
- * Queue selector
- */
-.node {
- cursor: pointer;
-}
-
-.node circle {
- fill: #fff;
- stroke: steelblue;
- stroke-width: 3px;
-}
-
-.node text {
- font: 12px sans-serif;
-}
-
-.link {
- fill: none;
- stroke: #ccc;
- stroke-width: 2px;
-}
-
-.lr-margin {
- margin: 0px 30px;
-}
-
-.footer {
- background-color: @white;
- color: @text-color;
-
- padding: 10px 0px;
- margin: 0px;
-
- border-top: 1px lightgrey solid;
-
- font-size: .9em;
-}
-
-.table {
- margin-bottom: 0px;
- border: none;
-
- overflow: hidden;
-}
-
-.table-bordered > thead > tr > th, .table-bordered > tbody > tr > th, .table-bordered > tfoot > tr > th, .table-bordered > thead > tr > td, .table-bordered > tbody > tr > td, .table-bordered > tfoot > tr > td {
- border: none !important;
-}
-
-.dataTables_wrapper .table {
- border: 1px solid lightgrey;
- border-bottom: 1px solid lightgrey !important;
- border-radious: 5px;
-}
-
-.dataTables_wrapper .table-bordered > thead > tr > th, .table-bordered > tbody > tr > th, .table-bordered > tfoot > tr > th, .table-bordered > thead > tr > td, .table-bordered > tbody > tr > td, .table-bordered > tfoot > tr > td {
- border: 1px solid lightgrey;
-}
-
-td {
- padding: 8px 15px 8px 15px !important;
-}
-
-.footer-frame {
- height: 60px;
-}
-.footer {
- height: 40px;
-}
-
-.footer-pusher {
- min-height: 100%;
- height: auto !important;
- height: 100%;
- margin: 0 auto -40px; // Must be same as footer & footer-frame
-}
-
-.panel {
- background-color: #fff;
- border: 1px solid #ddd;
- border-radius: 15px;
- -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
- box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
- overflow: hidden;
-}
-
-.panel-default .container-fluid {
- margin-top: -45px !important;
- margin-bottom: -10px !important;
-}
-
-.panel-default > .panel-heading {
- background-image: none;
- background-color: #f5f5f5 !important;
- border-radius: 15px 15px 0px 0px;
-}
-
-
-.panel-heading {
- background-color: rgba(0, 0, 0, 0) !important;
- font-weight: bold;
-}
-
-.hadoop-brand-image {
- margin-top: -10px;
- width: auto;
- height: 45px;
-}
-
-li a.navigation-link.ember-view {
- color: #2196f3;
- font-weight: bold;
-}
-
-.breadcrumb-bar .refresh {
- position: absolute;
- right: 20px;
- top: 3px;
-}
-
-.x-scroll {
- overflow-x: scroll;
-}
-
-.donut-chart {
- padding: 0px !important;
-}
-
-.donut-chart svg {
- width: 100%;
-}
-
-div.attempt-info-panel table {
- table-layout: fixed;
-}
-
-div.attempt-info-panel table > tbody > tr > td:last-of-type {
- overflow: hidden;
- text-overflow: ellipsis;
-}
-
-#main {
- float: left;
- width: 750px;
-}
-
-#sidebar {
- float: right;
- width: 100px;
-}
-
-#sequence {
- width: 600px;
- height: 70px;
-}
-
-#legend {
- padding: 10px 0 0 3px;
-}
-
-#sequence text, #legend text {
- font-weight: 600;
- fill: #fff;
-}
-
-#chart {
- position: relative;
-}
-
-#chart path {
- stroke: #fff;
-}
-
-#explanation {
- position: absolute;
- top: 360px;
- left: 385px;
- width: 230px;
- text-align: center;
- color: #666;
- z-index: -1;
-}
-
-.simple-bar-chart .axis text {
- font: 10px sans-serif;
-}
-
-.simple-bar-chart .axis path, .simple-bar-chart .axis line {
- fill: none;
- stroke: #000;
- stroke-width: 1px;
-}
-
-.simple-bar-chart rect {
- fill: steelblue;
-}
-
-.simple-bar-chart rect:hover {
- fill: brown;
- cursor: pointer;
-}
-
-.yarn-flow-runs .chart-panel, .yarn-flow-runinfo .chart-panel {
- overflow: auto;
-}
-
-.yarn-flow-runs .glyphicon-remove, .yarn-flow-runinfo .glyphicon-remove {
- cursor: pointer;
-}
-
-.yarn-flow-runs .glyphicon-remove:hover, .yarn-flow-runinfo .glyphicon-remove:hover {
- color: #c9302c;
-}
-
-.yarn-flow-runs .dropdown-menu .item-text, .yarn-flow-runinfo .dropdown-menu .item-text {
- width: auto;
- display: inline-block;
-}
-
-.yarn-flow-runs .dropdown-menu .item-icon, .yarn-flow-runinfo .dropdown-menu .item-icon {
- width: 14px;
- display: inline-block;
-}
-
-.yarn-cluster-status i,
-.em-table-simple-status-cell i{
- display: inline-block;
- border-radius: 100%;
- border: 1px solid;
-}
-
-.yarn-cluster-status i {
- width: 12px;
- height: 12px;
- margin: 3px;
- vertical-align: bottom;
-}
-
-.em-table-simple-status-cell i {
- width: 10px;
- height: 10px;
- margin: 3px 3px 3px 0;
- vertical-align: text-bottom;
-}
-
-.yarn-cluster-status i.started,
-.em-table-simple-status-cell i.finished {
- border-color: #43b135;
- background-color: #60cea5;
-}
-.yarn-cluster-status i.stopped,
-.em-table-simple-status-cell i.killed {
- border-color: #b04b4e;
- background-color: #ef6162;
-}
-.yarn-cluster-status i.inited,
-.em-table-simple-status-cell i.running {
- border-color: #1c95c0;
- background-color: #26bbf0;
-}
-.yarn-cluster-status i.notinited,
-.em-table-simple-status-cell i.accepted {
- border-color: #dca41b;
- background-color: #ffbc0b;
-}
-.yarn-cluster-info {
- display: flex;
- margin-left: auto
-}
-
-.yarn-ui-footer {
- display: flex;
- padding: 10px 25px;
-}
-
-.deploy-service textarea {
- border-radius: 5px !important;
- resize: none;
- word-wrap: break-word;
-}
-
-.deploy-service .loading-state {
- opacity: 0.5;
-}
-
-.deploy-service .loading-state img {
- width: 80px;
- height: 80px;
- margin: 40px auto;
- left: 50% !important;
- position: absolute;
- z-index: 9999;
-}
-
-.align-center {
- text-align: center !important;
-}
-
-.bold-text {
- font-weight: bold !important;
-}
-
-.deploy-service .saved-list {
- min-height: 600px;
-}
-
-.deploy-service .glyphicon {
- cursor: pointer;
-}
-
-.deploy-service .remove-icon:hover {
- color: #d9534f;
-}
-
-.deploy-service .savedlist-column {
- padding-top: 10px;
-}
-
-.deploy-service .definition-column {
- padding-top: 10px;
- border-left: 1px solid #ddd;
-}
-
-.deploy-service .content-area {
- padding: 15px 0px;
- border-top: 1px solid #ddd;
-}
-
-.deploy-service .custom-json-area {
- padding: 10px 0;
- margin-top: -26px;
-}
-
-.deploy-service-modal .modal-dialog {
- width: 400px;
-}
-
-.deploy-service-modal .form-group {
- margin-bottom: 5px;
-}
-
-.deploy-service .action-btns {
- text-align: right;
- padding-bottom: 15px;
- padding-right: 0;
-}
-
-table.table-custom-action > thead > tr > th:last-of-type, table.table-custom-action > tbody > tr > td:last-of-type {
- width: 50px !important;
-}
-
-.deploy-service .toggle-btn.active {
- color: #fff;
- background-color: #337ab7;
- border-color: #337ab7;
- text-shadow: none;
-}
-
-.deploy-service .service-resp {
- word-wrap: break-word;
-}
-
-table.table-custom-bordered {
- border: 1px solid #ddd !important;
- border-radius: 4px !important;
-}
-
-table.table-custom-bordered > thead > tr > th, table.table-custom-bordered > tbody > tr > td {
- border-bottom: 1px solid #ddd !important;
- border-right: 1px solid #ddd !important;
-}
-
-table.table-custom-striped > thead > tr, .table-custom-striped > tbody > tr:nth-of-type(even) {
- background-color: #f9f9f9 !important;
-}
-
-table.table-custom-header > thead > tr > th {
- background-color: #f5f5f5 !important;
-}
-
-table.table-radius-none {
- border-radius: 0 !important;
-}
-
-table.table-border-none {
- border: none !important;
-}
-
-.deploy-service label.required:after, .deploy-service-modal label.required:after {
- content: '*';
- color: #d9534f;
-}
-
-.deploy-service .form-group.shrink-height {
- margin-bottom: -8px;
-}
-
-table.fix-table-overflow {
- table-layout: fixed;
-}
-
-table.fix-table-overflow > tbody > tr > td:last-of-type {
- overflow: hidden;
- text-overflow: ellipsis;
-}
-
-div.tooltip.info-tooltip {
- font: 14px sans-serif !important;
- background: lightsteelblue;
- word-wrap: break-word;
- position: absolute;
- text-align: justify;
- border-radius: 3px;
- z-index: 9999;
- padding: 10px;
- display: none;
- min-width: 200px;
- max-width: 500px;
- opacity: 1;
- top: 10px;
- left: 0;
-}
-
-div.tooltip.info-tooltip > span.top-arrow {
- color: lightsteelblue;
- position: absolute;
- top: -10px;
- left: 10px;
-}
-
-span.info-icon {
- color: #337ab7 !important;
-}
-
-div.service-action-mask {
- position: absolute;
- opacity: 0.5;
- z-index: 9999;
- width: 100%;
- height: 100%;
-}
-
-div.service-action-mask img {
- position: absolute;
- width: 80px;
- height: 80px;
- margin: 40px auto;
- left: 45% !important;
- z-index: 9999;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel {
- width: 240px;
- margin-right: 15px;
- padding: 0 0 15px 0;
- background: none;
- border: none;
- border-radius: 0;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .field-list > li .em-table-facet-panel-values{
- margin-top: 0px;
- margin-bottom: -4px;
- background: #fff;
- border: 1px solid #d5d5d5;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .field-name {
- background-color: #f7f7f7;
- border-bottom: 1px solid #d5d5d5;
- padding: 10px 15px;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .all-button {
- right: 5px !important;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .field-name::before {
- margin-right: 5px;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .value-list {
- padding: 0 10px;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .filter-box {
- padding: 5px;
- margin: 5px 0;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .more-less {
- padding: 5px 0;
-}
-
-.yarn-applications-container .table-panel-left .em-table-facet-panel .field-list {
- padding-top: 0px;
-}
-
-.yarn-applications-container .em-table .table-column .table-header-cell {
- padding: 7.5px 0 7.5px 7.5px;
-}
-
-.yarn-applications-container .ember-view.table-cell {
- padding: 10px 15px;
- height: auto;
-}
-
-.yarn-applications-container .resize-column {
- position: absolute;
- right: -14px;
- display: inline-block;
- padding: 10px 0;
- top: -36px;
- font-size: 42px;
- line-height: 2em;
-}
-
-.em-table .table-column .table-header-cell {
- background-color: #f7f7f7;
- border-bottom: 1px solid #d5d5d5;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
new file mode 100644
index 0000000..350c794
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/app.scss
@@ -0,0 +1,723 @@
+@import 'variables.scss';
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+body, html, body > .ember-view {
+ height: 100%;
+ overflow: visible;
+ background: $yarn-bg;
+ color: $yarn-primary-color;
+ font-family: "Open Sans","Helvetica Neue", sans-serif!important;
+}
+body, html {
+ min-width: 1024px;
+}
+
+/*
+ Over all style
+ */
+text {
+ font-size: 14px;
+}
+
+text.small {
+ font-size: 8px;
+}
+
+html, body
+{
+ margin: 0px;
+ padding: 0px;
+ height: 100%;
+ width: 100%;
+}
+
+
+/*
+ queue's style (left banner of queues)
+ */
+
+text.queue {
+ font-family : sans-serif;
+ font-size : 15px;
+ fill : gray;
+}
+
+text.heatmap-cell {
+ font-size: 14px;
+ text-anchor: middle;
+ fill: Azure;
+ text-align: center;
+}
+
+.hyperlink {
+ cursor: pointer;
+ -webkit-user-select: none;
+ -moz-user-select: none;
+ -ms-user-select: none;
+ user-select: none;
+}
+
+.heatmap-clear {
+ fill: #337ab7;
+}
+
+text.heatmap-cell-notselected {
+ font: 14px sans-serif;
+ font-weight: bold;
+ text-anchor: middle;
+ fill: Silver;
+ text-align: center;
+
+}
+
+text.heatmap-rack {
+ font-size: 18px;
+ font-weight: 400;
+ fill: #4b5052;
+}
+
+path.queue {
+ stroke: "red";
+ fill: none;
+}
+
+/*
+ background style
+ */
+line.grid {
+ stroke: WhiteSmoke;
+}
+
+line.chart {
+ stroke: Gray;
+}
+
+/*
+ charts styles
+ */
+text.chart-title {
+ font-size: 30px;
+ text-anchor: middle;
+ fill: #4b5052;
+}
+
+text.donut-highlight-text, text.donut-highlight-sub {
+ font-size: 15px;
+ font-family: sans-serif;
+ text-anchor: middle;
+ fill: Gray;
+ vertical-align: middle;
+}
+
+text.donut-highlight-sub {
+ font-size: 23px;
+ margin-top: 10px;
+}
+
+rect.chart-frame {
+ fill: none;
+}
+
+text.bar-chart-text {
+ font-size: 8px;
+ font-family: sans-serif;
+ vertical-align: middle;
+ fill: Gray;;
+}
+
+div.tooltip {
+ position: absolute;
+ text-align: center;
+ padding: 10px;
+ font-size: 16px;
+ background: black;
+ color: white;
+ border: 0px;
+ border-radius: 8px;
+ pointer-events: none;
+}
+
+/*
+ * Loading icon styles
+ */
+.loading-mask > img {
+ display: block;
+ margin: 0 auto;
+}
+
+/*
+ * Data table
+ */
+
+table.dataTable thead .sorting {
+ background-image: url("/assets/images/datatables/sort_both.png");
+}
+table.dataTable thead .sorting_asc {
+ background-image: url("/assets/images/datatables/sort_asc.png");
+}
+table.dataTable thead .sorting_desc {
+ background-image: url("/assets/images/datatables/sort_desc.png");
+}
+table.dataTable thead .sorting_asc_disabled {
+ background-image: url("/assets/images/datatables/sort_asc_disabled.png");
+}
+table.dataTable thead .sorting_desc_disabled {
+ background-image: url("/assets/images/datatables/sort_desc_disabled.png");
+}
+
+.add-ellipsis {
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+.breadcrumb {
+ padding-bottom: 3px;
+ background-color: none;
+}
+
+.navbar-default .navbar-nav > li > a {
+ color: #337ab7;
+}
+
+/*
+ * Queue selector
+ */
+.node {
+ cursor: pointer;
+}
+
+.node circle {
+ fill: $yarn-panel-bg;
+ stroke: steelblue;
+ stroke-width: 3px;
+}
+
+.node text {
+ font: 12px sans-serif;
+}
+
+.link {
+ fill: none;
+ stroke: #ccc;
+ stroke-width: 2px;
+}
+
+.lr-margin {
+ margin: 0px 30px;
+}
+
+.footer {
+ background-color: $yarn-panel-bg;
+ color: $yarn-primary-color;
+
+ padding: 10px 0px;
+ margin: 0px;
+
+ border-top: 1px lightgrey solid;
+
+ font-size: .9em;
+}
+
+.table {
+ margin-bottom: 0px;
+ border: none;
+
+ overflow: hidden;
+}
+
+.table-bordered > thead > tr > th, .table-bordered > tbody > tr > th, .table-bordered > tfoot > tr > th, .table-bordered > thead > tr > td, .table-bordered > tbody > tr > td, .table-bordered > tfoot > tr > td {
+ border: none !important;
+}
+
+.dataTables_wrapper .table {
+ border: 1px solid lightgrey;
+ border-bottom: 1px solid lightgrey !important;
+ border-radius: 5px;
+}
+
+.dataTables_wrapper .table-bordered > thead > tr > th, .table-bordered > tbody > tr > th, .table-bordered > tfoot > tr > th, .table-bordered > thead > tr > td, .table-bordered > tbody > tr > td, .table-bordered > tfoot > tr > td {
+ border: 1px solid lightgrey;
+}
+
+td {
+ padding: 8px 15px 8px 15px !important;
+}
+
+.footer-frame {
+ height: 60px;
+}
+.footer {
+ height: 40px;
+}
+
+.footer-pusher {
+ min-height: 100%;
+ height: auto !important;
+ height: 100%;
+ margin: 0 auto -40px; // Must be same as footer & footer-frame
+}
+
+.panel {
+ background-color: $yarn-panel-bg;
+ border: 1px solid $yarn-border-color;
+ border-radius: 3px;
+ -webkit-box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
+ box-shadow: 0 1px 1px rgba(0, 0, 0, 0.05);
+ overflow: hidden;
+}
+
+.panel-default .container-fluid {
+ margin-top: -45px !important;
+ margin-bottom: -10px !important;
+}
+
+.panel-default > .panel-heading {
+ background-image: none;
+ background-color: $yarn-header-bg !important;
+ border-color: $yarn-border-color;
+ border-radius: 3px;
+}
+
+
+.panel-heading {
+ background-color: rgba(0, 0, 0, 0) !important;
+ font-weight: bold;
+}
+
+.hadoop-brand-image {
+ margin-top: -10px;
+ width: auto;
+ height: 45px;
+}
+
+li a.navigation-link.ember-view {
+ color: #2196f3;
+ font-weight: bold;
+}
+
+.breadcrumb-bar .refresh {
+ position: absolute;
+ right: 20px;
+ top: 3px;
+}
+
+.x-scroll {
+ overflow-x: scroll;
+}
+
+.donut-chart {
+ padding: 0px !important;
+}
+
+.donut-chart svg {
+ width: 100%;
+}
+
+div.attempt-info-panel table {
+ table-layout: fixed;
+}
+
+div.attempt-info-panel table > tbody > tr > td:last-of-type {
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+#main {
+ float: left;
+ width: 750px;
+}
+
+#sidebar {
+ float: right;
+ width: 100px;
+}
+
+#sequence {
+ width: 600px;
+ height: 70px;
+}
+
+#legend {
+ padding: 10px 0 0 3px;
+}
+
+#sequence text, #legend text {
+ font-weight: 600;
+ fill: $yarn-panel-bg;
+}
+
+#chart {
+ position: relative;
+}
+
+#chart path {
+ stroke: $yarn-panel-bg;
+}
+
+#explanation {
+ position: absolute;
+ top: 360px;
+ left: 385px;
+ width: 230px;
+ text-align: center;
+ color: #666;
+ z-index: -1;
+}
+
+.simple-bar-chart .axis text {
+ font: 10px sans-serif;
+}
+
+.simple-bar-chart .axis path, .simple-bar-chart .axis line {
+ fill: none;
+ stroke: #000;
+ stroke-width: 1px;
+}
+
+.simple-bar-chart rect {
+ fill: steelblue;
+}
+
+.simple-bar-chart rect:hover {
+ fill: brown;
+ cursor: pointer;
+}
+
+.yarn-flow-runs .chart-panel, .yarn-flow-runinfo .chart-panel {
+ overflow: auto;
+}
+
+.yarn-flow-runs .glyphicon-remove, .yarn-flow-runinfo .glyphicon-remove {
+ cursor: pointer;
+}
+
+.yarn-flow-runs .glyphicon-remove:hover, .yarn-flow-runinfo .glyphicon-remove:hover {
+ color: #c9302c;
+}
+
+.yarn-flow-runs .dropdown-menu .item-text, .yarn-flow-runinfo .dropdown-menu .item-text {
+ width: auto;
+ display: inline-block;
+}
+
+.yarn-flow-runs .dropdown-menu .item-icon, .yarn-flow-runinfo .dropdown-menu .item-icon {
+ width: 14px;
+ display: inline-block;
+}
+
+.yarn-cluster-status i,
+.em-table-simple-status-cell i{
+ display: inline-block;
+ border-radius: 100%;
+ border: 1px solid;
+}
+
+.yarn-cluster-status i {
+ width: 12px;
+ height: 12px;
+ margin: 3px;
+ vertical-align: bottom;
+}
+
+.em-table-simple-status-cell i {
+ width: 10px;
+ height: 10px;
+ margin: 3px 3px 3px 0;
+ vertical-align: text-bottom;
+}
+
+.yarn-cluster-status i.started,
+.em-table-simple-status-cell i.finished {
+ border-color: $yarn-success-border;
+ background-color: $yarn-success-bg;
+}
+
+.yarn-cluster-status i.stopped,
+.em-table-simple-status-cell i.killed,
+.em-table-simple-status-cell i.failed {
+ border-color: $yarn-error-border;
+ background-color: $yarn-error-bg;
+}
+.yarn-cluster-status i.inited,
+.em-table-simple-status-cell i.running {
+ border-color: $yarn-info-border;
+ background-color: $yarn-info-bg;
+}
+.yarn-cluster-status i.notinited,
+.em-table-simple-status-cell i.accepted {
+ border-color: $yarn-warn-border;
+ background-color: $yarn-warn-bg;
+}
+.yarn-cluster-info {
+ display: flex;
+ margin-left: auto
+}
+
+.yarn-ui-footer {
+ display: flex;
+ padding: 10px 25px;
+}
+
+.deploy-service textarea {
+ border-radius: 5px !important;
+ resize: none;
+ word-wrap: break-word;
+}
+
+.deploy-service .loading-state {
+ opacity: 0.5;
+}
+
+.deploy-service .loading-state img {
+ width: 80px;
+ height: 80px;
+ margin: 40px auto;
+ left: 50% !important;
+ position: absolute;
+ z-index: 9999;
+}
+
+.align-center {
+ text-align: center !important;
+}
+
+.bold-text {
+ font-weight: bold !important;
+}
+
+.deploy-service .saved-list {
+ min-height: 600px;
+}
+
+.deploy-service .glyphicon {
+ cursor: pointer;
+}
+
+.deploy-service .remove-icon:hover {
+ color: #d9534f;
+}
+
+.deploy-service .savedlist-column {
+ padding-top: 10px;
+}
+
+.deploy-service .definition-column {
+ padding-top: 10px;
+ border-left: 1px solid #ddd;
+}
+
+.deploy-service .content-area {
+ padding: 15px 0px;
+ border-top: 1px solid #ddd;
+}
+
+.deploy-service .custom-json-area {
+ padding: 10px 0;
+ margin-top: -26px;
+}
+
+.deploy-service-modal .modal-dialog {
+ width: 400px;
+}
+
+.deploy-service-modal .form-group {
+ margin-bottom: 5px;
+}
+
+.deploy-service .action-btns {
+ text-align: right;
+ padding-bottom: 15px;
+ padding-right: 0;
+}
+
+table.table-custom-action > thead > tr > th:last-of-type, table.table-custom-action > tbody > tr > td:last-of-type {
+ width: 50px !important;
+}
+
+.deploy-service .toggle-btn.active {
+ color: $yarn-panel-bg;
+ background-color: #337ab7;
+ border-color: #337ab7;
+ text-shadow: none;
+}
+
+.deploy-service .service-resp {
+ word-wrap: break-word;
+}
+
+table.table-custom-bordered {
+ border: 1px solid #ddd !important;
+ border-radius: 4px !important;
+}
+
+table.table-custom-bordered > thead > tr > th, table.table-custom-bordered > tbody > tr > td {
+ border-bottom: 1px solid #ddd !important;
+ border-right: 1px solid #ddd !important;
+}
+
+table.table-custom-striped > thead > tr, .table-custom-striped > tbody > tr:nth-of-type(even) {
+ background-color: #f9f9f9 !important;
+}
+
+table.table-custom-header > thead > tr > th {
+ background-color: #f5f5f5 !important;
+}
+
+table.table-radius-none {
+ border-radius: 0 !important;
+}
+
+table.table-border-none {
+ border: none !important;
+}
+
+.deploy-service label.required:after, .deploy-service-modal label.required:after {
+ content: '*';
+ color: #d9534f;
+}
+
+.deploy-service .form-group.shrink-height {
+ margin-bottom: -8px;
+}
+
+table.fix-table-overflow {
+ table-layout: fixed;
+}
+
+table.fix-table-overflow > tbody > tr > td:last-of-type {
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+div.tooltip.info-tooltip {
+ font: 14px sans-serif !important;
+ background: lightsteelblue;
+ word-wrap: break-word;
+ position: absolute;
+ text-align: justify;
+ border-radius: 3px;
+ z-index: 9999;
+ padding: 10px;
+ display: none;
+ min-width: 200px;
+ max-width: 500px;
+ opacity: 1;
+ top: 10px;
+ left: 0;
+}
+
+div.tooltip.info-tooltip > span.top-arrow {
+ color: lightsteelblue;
+ position: absolute;
+ top: -10px;
+ left: 10px;
+}
+
+span.info-icon {
+ color: #337ab7 !important;
+}
+
+div.service-action-mask {
+ position: absolute;
+ opacity: 0.5;
+ z-index: 9999;
+ width: 100%;
+ height: 100%;
+}
+
+div.service-action-mask img {
+ position: absolute;
+ width: 80px;
+ height: 80px;
+ margin: 40px auto;
+ left: 45% !important;
+ z-index: 9999;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel {
+ width: 240px;
+ margin-right: 15px;
+ padding: 0 0 15px 0;
+ background: none;
+ border: none;
+ border-radius: 0;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .field-list > li .em-table-facet-panel-values{
+ margin-top: 0px;
+ margin-bottom: -4px;
+ background: $yarn-panel-bg;
+ border: 1px solid $yarn-border-color;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .field-name {
+ background-color: $yarn-header-bg;
+ border-bottom: 1px solid $yarn-border-color;
+ padding: 10px 15px;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .all-button {
+ right: 5px !important;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .field-name::before {
+ margin-right: 5px;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .value-list {
+ padding: 0 10px;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .filter-box {
+ padding: 5px;
+ margin: 5px 0;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .more-less {
+ padding: 5px 0;
+}
+
+.yarn-applications-container .table-panel-left .em-table-facet-panel .field-list {
+ padding-top: 0px;
+}
+
+.yarn-applications-container .em-table .table-column .table-header-cell {
+ padding: 7.5px 0 7.5px 7.5px;
+}
+
+.yarn-applications-container .ember-view.table-cell {
+ padding: 10px 15px;
+ height: auto;
+}
+
+.yarn-applications-container .resize-column {
+ position: absolute;
+ right: -14px;
+ display: inline-block;
+ padding: 10px 0;
+ top: -36px;
+ font-size: 42px;
+ line-height: 2em;
+}
+
+.em-table .table-column .table-header-cell {
+ background-color: $yarn-header-bg;
+ border-bottom: 1px solid $yarn-border-color;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/colors.scss
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/colors.scss b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/colors.scss
new file mode 100644
index 0000000..5d74b86
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/colors.scss
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Colors
+$color-ghost-white: #f1f2f8;
+$color-white: #fff;
+
+$color-gray-20: #333;
+$color-gray-97: #f7f7f7;
+$color-light-gray: #d5d5d5;
+
+$color-green-primary: #60cea5;
+$color-green-secondary: #43b135;
+
+$color-red-primary: #ef6162;
+$color-red-secondary: #b04b4e;
+
+$color-blue-primary: #26bbf0;
+$color-blue-secondary:#1c95c0;
+
+$color-yellow-primary: #ffbc0b;
+$color-yellow-secondary: #dca41b;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/variables.scss
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/variables.scss b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/variables.scss
new file mode 100644
index 0000000..377ac00
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/styles/variables.scss
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+@import './colors.scss';
+
+$yarn-bg: $color-ghost-white;
+$yarn-panel-bg: $color-white;
+$yarn-primary-color: $color-gray-20;
+
+$yarn-header-bg: $color-gray-97;
+$yarn-border-color: $color-light-gray;
+
+
+$yarn-success-border: $color-green-secondary;
+$yarn-success-bg: $color-green-primary;
+$yarn-error-border: $color-red-secondary;
+$yarn-error-bg: $color-red-primary;
+$yarn-info-border: $color-blue-secondary;
+$yarn-info-bg: $color-blue-primary;
+$yarn-warn-border: $color-yellow-secondary;
+$yarn-warn-bg: $color-yellow-primary;
+
+
+//shadows
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
index af0cdf4..f3acad0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
@@ -23,10 +23,10 @@ export default {
"#dbdb8d", "#17becf", "#9edae5"],
colorMap: {
- "warn": "#ff7f0e",
- "good": "#2ca02c",
- "error": "#d62728",
- "others": "#7f7f7f",
+ "warn": "#ffbc0b",
+ "good": "#60cea5",
+ "error": "#cc6164",
+ "others": "#26bbf0",
},
getColors: function(nColors, colorsTarget, reverse = false) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
index 9b2c1e4..234d0f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package.json
@@ -52,7 +52,8 @@
"ember-truth-helpers": "1.3.0",
"loader.js": "4.2.3",
"select2": "4.0.0",
- "testem": "0.9.11"
+ "testem": "0.9.11",
+ "ember-cli-sass": "=7.0.0"
},
"dependencies": {
"em-helpers": "^0.8.0",
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/19] hadoop git commit: YARN-7419. CapacityScheduler: Allow auto
leaf queue creation after queue mapping. (Suma Shivaprasad via wangda)
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
index 0a8d6fe..80b7f2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.server.resourcemanager.placement
+ .ApplicationPlacementContext;
public class AppAddedSchedulerEvent extends SchedulerEvent {
@@ -31,15 +33,23 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
private final ReservationId reservationID;
private final boolean isAppRecovering;
private final Priority appPriority;
+ private final ApplicationPlacementContext placementContext;
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
String user) {
- this(applicationId, queue, user, false, null, Priority.newInstance(0));
+ this(applicationId, queue, user, false, null, Priority.newInstance(0),
+ null);
+ }
+
+ public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
+ String user, ApplicationPlacementContext placementContext) {
+ this(applicationId, queue, user, false, null, Priority.newInstance(0),
+ placementContext);
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
String user, ReservationId reservationID, Priority appPriority) {
- this(applicationId, queue, user, false, reservationID, appPriority);
+ this(applicationId, queue, user, false, reservationID, appPriority, null);
}
public AppAddedSchedulerEvent(String user,
@@ -47,12 +57,20 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
Priority appPriority) {
this(submissionContext.getApplicationId(), submissionContext.getQueue(),
user, isAppRecovering, submissionContext.getReservationID(),
- appPriority);
+ appPriority, null);
+ }
+
+ public AppAddedSchedulerEvent(String user,
+ ApplicationSubmissionContext submissionContext, boolean isAppRecovering,
+ Priority appPriority, ApplicationPlacementContext placementContext) {
+ this(submissionContext.getApplicationId(), submissionContext.getQueue(),
+ user, isAppRecovering, submissionContext.getReservationID(),
+ appPriority, placementContext);
}
public AppAddedSchedulerEvent(ApplicationId applicationId, String queue,
String user, boolean isAppRecovering, ReservationId reservationID,
- Priority appPriority) {
+ Priority appPriority, ApplicationPlacementContext placementContext) {
super(SchedulerEventType.APP_ADDED);
this.applicationId = applicationId;
this.queue = queue;
@@ -60,6 +78,7 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
this.reservationID = reservationID;
this.isAppRecovering = isAppRecovering;
this.appPriority = appPriority;
+ this.placementContext = placementContext;
}
public ApplicationId getApplicationId() {
@@ -85,4 +104,8 @@ public class AppAddedSchedulerEvent extends SchedulerEvent {
public Priority getApplicatonPriority() {
return appPriority;
}
+
+ public ApplicationPlacementContext getPlacementContext() {
+ return placementContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8179321..9445fa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.MockApps;
@@ -70,6 +69,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
@@ -84,7 +84,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptI
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -856,28 +855,32 @@ public class TestAppManager{
Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
-
+
@Test
public void testRMAppSubmitWithQueueChanged() throws Exception {
// Setup a PlacementManager returns a new queue
PlacementManager placementMgr = mock(PlacementManager.class);
- doAnswer(new Answer<Void>() {
+ doAnswer(new Answer<ApplicationPlacementContext>() {
@Override
- public Void answer(InvocationOnMock invocation) throws Throwable {
- ApplicationSubmissionContext ctx =
- (ApplicationSubmissionContext) invocation.getArguments()[0];
- ctx.setQueue("newQueue");
- return null;
+ public ApplicationPlacementContext answer(InvocationOnMock invocation)
+ throws Throwable {
+ return new ApplicationPlacementContext("newQueue");
}
-
- }).when(placementMgr).placeApplication(any(ApplicationSubmissionContext.class),
- any(String.class));
+
+ }).when(placementMgr).placeApplication(
+ any(ApplicationSubmissionContext.class), any(String.class));
rmContext.setQueuePlacementManager(placementMgr);
-
+
asContext.setQueue("oldQueue");
appMonitor.submitApplication(asContext, "test");
+
RMApp app = rmContext.getRMApps().get(appId);
+ RMAppEvent event = new RMAppEvent(appId, RMAppEventType.START);
+ rmContext.getRMApps().get(appId).handle(event);
+ event = new RMAppEvent(appId, RMAppEventType.APP_NEW_SAVED);
+ rmContext.getRMApps().get(appId).handle(event);
+
Assert.assertNotNull("app is null", app);
Assert.assertEquals("newQueue", asContext.getQueue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
index 61bc8d9..864acc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/placement/TestUserGroupMappingPlacementRule.java
@@ -52,14 +52,14 @@ public class TestUserGroupMappingPlacementRule {
private void verifyQueueMapping(QueueMapping queueMapping, String inputUser,
String inputQueue, String expectedQueue, boolean overwrite) throws YarnException {
Groups groups = new Groups(conf);
- UserGroupMappingPlacementRule rule =
- new UserGroupMappingPlacementRule(overwrite, Arrays.asList(queueMapping),
- groups);
- ApplicationSubmissionContext asc =
- Records.newRecord(ApplicationSubmissionContext.class);
+ UserGroupMappingPlacementRule rule = new UserGroupMappingPlacementRule(
+ overwrite, Arrays.asList(queueMapping), groups);
+ ApplicationSubmissionContext asc = Records.newRecord(
+ ApplicationSubmissionContext.class);
asc.setQueue(inputQueue);
- String queue = rule.getQueueForApp(asc, inputUser);
- Assert.assertEquals(expectedQueue, queue);
+ ApplicationPlacementContext ctx = rule.getPlacementForApp(asc, inputUser);
+ Assert.assertEquals(expectedQueue,
+ ctx != null ? ctx.getQueue() : inputQueue);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index cdc67ed..cb1f794 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -803,6 +803,7 @@ public class TestSchedulerUtils {
Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
EventHandler<SchedulerEvent> handler, String queueName)
throws Exception {
+
ApplicationId appId =
ApplicationId.newInstance(System.currentTimeMillis(), 1);
AppAddedSchedulerEvent appAddedEvent =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 800d023..1edb0da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -103,6 +103,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -904,7 +905,7 @@ public class TestCapacityScheduler {
(B3_CAPACITY/100.0f) * capB, 1.0f, 1.0f);
}
- private void checkQueueCapacity(CSQueue q, float expectedCapacity,
+ void checkQueueCapacity(CSQueue q, float expectedCapacity,
float expectedAbsCapacity, float expectedMaxCapacity,
float expectedAbsMaxCapacity) {
final float epsilon = 1e-5f;
@@ -917,7 +918,7 @@ public class TestCapacityScheduler {
q.getAbsoluteMaximumCapacity(), epsilon);
}
- private CSQueue findQueue(CSQueue root, String queuePath) {
+ CSQueue findQueue(CSQueue root, String queuePath) {
if (root.getQueuePath().equals(queuePath)) {
return root;
}
@@ -1396,7 +1397,6 @@ public class TestCapacityScheduler {
AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
(AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
.getResourceScheduler();
-
SchedulerApplication<SchedulerApplicationAttempt> app =
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
cs.getSchedulerApplications(), cs, "a1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
new file mode 100644
index 0000000..7090bc9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -0,0 +1,794 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.CURRENT_USER_MAPPING;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueueUtils.EPSILON;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for creation and reinitilization of auto created leaf queues
+ * under a ManagedParentQueue.
+ */
+public class TestCapacitySchedulerAutoQueueCreation {
+
+ private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
+ private final int GB = 1024;
+ private final static ContainerUpdates NULL_UPDATE_REQUESTS =
+ new ContainerUpdates();
+
+ private static final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ private static final String B = CapacitySchedulerConfiguration.ROOT + ".b";
+ private static final String C = CapacitySchedulerConfiguration.ROOT + ".c";
+ private static final String D = CapacitySchedulerConfiguration.ROOT + ".d";
+ private static final String A1 = A + ".a1";
+ private static final String A2 = A + ".a2";
+ private static final String B1 = B + ".b1";
+ private static final String B2 = B + ".b2";
+ private static final String B3 = B + ".b3";
+ private static final String C1 = C + ".c1";
+ private static final String C2 = C + ".c2";
+ private static final String C3 = C + ".c3";
+ private static float A_CAPACITY = 20f;
+ private static float B_CAPACITY = 40f;
+ private static float C_CAPACITY = 20f;
+ private static float D_CAPACITY = 20f;
+ private static float A1_CAPACITY = 30;
+ private static float A2_CAPACITY = 70;
+ private static float B1_CAPACITY = 60f;
+ private static float B2_CAPACITY = 20f;
+ private static float B3_CAPACITY = 20f;
+ private static float C1_CAPACITY = 20f;
+ private static float C2_CAPACITY = 20f;
+
+ private static String USER = "user_";
+ private static String USER0 = USER + 0;
+ private static String USER2 = USER + 2;
+ private static String PARENT_QUEUE = "c";
+
+ private MockRM mockRM = null;
+
+ private CapacityScheduler cs;
+
+ private final TestCapacityScheduler tcs = new TestCapacityScheduler();
+
+ private static SpyDispatcher dispatcher;
+
+ private static EventHandler<Event> rmAppEventEventHandler;
+
+ private static class SpyDispatcher extends AsyncDispatcher {
+
+ private static BlockingQueue<Event> eventQueue =
+ new LinkedBlockingQueue<>();
+
+ private static class SpyRMAppEventHandler implements EventHandler<Event> {
+ public void handle(Event event) {
+ eventQueue.add(event);
+ }
+ }
+
+ @Override
+ protected void dispatch(Event event) {
+ eventQueue.add(event);
+ }
+
+ @Override
+ public EventHandler<Event> getEventHandler() {
+ return rmAppEventEventHandler;
+ }
+
+ void spyOnNextEvent(Event expectedEvent, long timeout)
+ throws InterruptedException {
+
+ Event event = eventQueue.poll(timeout, TimeUnit.MILLISECONDS);
+ assertEquals(expectedEvent.getType(), event.getType());
+ assertEquals(expectedEvent.getClass(), event.getClass());
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ List<String> queuePlacementRules = new ArrayList<>();
+ queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
+ conf.setQueuePlacementRules(queuePlacementRules);
+
+ setupQueueMappings(conf);
+
+ mockRM = new MockRM(conf);
+ cs = (CapacityScheduler) mockRM.getResourceScheduler();
+
+ dispatcher = new SpyDispatcher();
+ rmAppEventEventHandler = new SpyDispatcher.SpyRMAppEventHandler();
+ dispatcher.register(RMAppEventType.class, rmAppEventEventHandler);
+ cs.updatePlacementRules();
+ mockRM.start();
+
+ cs.start();
+ }
+
+ private CapacitySchedulerConfiguration setupQueueMappings(
+ CapacitySchedulerConfiguration conf) {
+
+ //set queue mapping
+ List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
+ new ArrayList<>();
+ for (int i = 0; i <= 3; i++) {
+ //Set C as parent queue name for auto queue creation
+ UserGroupMappingPlacementRule.QueueMapping userQueueMapping =
+ new UserGroupMappingPlacementRule.QueueMapping(
+ UserGroupMappingPlacementRule.QueueMapping.MappingType.USER,
+ USER + i, getQueueMapping(PARENT_QUEUE, USER + i));
+ queueMappings.add(userQueueMapping);
+ }
+
+ conf.setQueueMappings(queueMappings);
+ //override with queue mappings
+ conf.setOverrideWithQueueMappings(true);
+ return conf;
+ }
+
+ /**
+ * @param conf, to be modified
+ * @return, CS configuration which has C
+ * as an auto creation enabled parent queue
+ * <p>
+ * root
+ * / \ \ \
+ * a b c d
+ * / \ / | \
+ * a1 a2 b1 b2 b3
+ */
+ private CapacitySchedulerConfiguration setupQueueConfiguration(
+ CapacitySchedulerConfiguration conf) {
+
+ //setup new queues with one of them auto enabled
+ // Define top-level queues
+ // Set childQueue for root
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT,
+ new String[] { "a", "b", "c", "d" });
+
+ conf.setCapacity(A, A_CAPACITY);
+ conf.setCapacity(B, B_CAPACITY);
+ conf.setCapacity(C, C_CAPACITY);
+ conf.setCapacity(D, D_CAPACITY);
+
+ // Define 2nd-level queues
+ conf.setQueues(A, new String[] { "a1", "a2" });
+ conf.setCapacity(A1, A1_CAPACITY);
+ conf.setUserLimitFactor(A1, 100.0f);
+ conf.setCapacity(A2, A2_CAPACITY);
+ conf.setUserLimitFactor(A2, 100.0f);
+
+ conf.setQueues(B, new String[] { "b1", "b2", "b3" });
+ conf.setCapacity(B1, B1_CAPACITY);
+ conf.setUserLimitFactor(B1, 100.0f);
+ conf.setCapacity(B2, B2_CAPACITY);
+ conf.setUserLimitFactor(B2, 100.0f);
+ conf.setCapacity(B3, B3_CAPACITY);
+ conf.setUserLimitFactor(B3, 100.0f);
+
+ conf.setUserLimitFactor(C, 1.0f);
+ conf.setAutoCreateChildQueueEnabled(C, true);
+
+ //Setup leaf queue template configs
+ conf.setAutoCreatedLeafQueueTemplateCapacity(C, 50.0f);
+ conf.setAutoCreatedLeafQueueTemplateMaxCapacity(C, 100.0f);
+
+ LOG.info("Setup " + C + " as an auto leaf creation enabled parent queue");
+
+ conf.setUserLimitFactor(D, 1.0f);
+ conf.setAutoCreateChildQueueEnabled(D, true);
+
+ //Setup leaf queue template configs
+ conf.setAutoCreatedLeafQueueTemplateCapacity(D, 10.0f);
+ conf.setAutoCreatedLeafQueueTemplateMaxCapacity(D, 100.0f);
+
+ LOG.info("Setup " + D + " as an auto leaf creation enabled parent queue");
+
+ return conf;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (mockRM != null) {
+ mockRM.stop();
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testAutoCreateLeafQueueCreation() throws Exception {
+
+ try {
+ // submit an app
+ submitApp(cs, USER0, USER0, PARENT_QUEUE);
+
+ // check preconditions
+ List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
+ assertEquals(1, appsInC.size());
+ assertNotNull(cs.getQueue(USER0));
+
+ AutoCreatedLeafQueue autoCreatedLeafQueue =
+ (AutoCreatedLeafQueue) cs.getQueue(USER0);
+ ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(
+ PARENT_QUEUE);
+ assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
+ validateCapacities(autoCreatedLeafQueue);
+ } finally {
+ cleanupQueue(USER0);
+ }
+ }
+
+ @Test
+ public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception {
+
+ try {
+ String host = "127.0.0.1";
+ RMNode node = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1,
+ host);
+ cs.handle(new NodeAddedSchedulerEvent(node));
+
+ // submit an app
+
+ RMApp app = mockRM.submitApp(GB, "test-auto-queue-creation-1", USER0,
+ null, USER0);
+ // check preconditions
+ List<ApplicationAttemptId> appsInC = cs.getAppsInQueue(PARENT_QUEUE);
+ assertEquals(1, appsInC.size());
+
+ assertNotNull(cs.getQueue(USER0));
+
+ AutoCreatedLeafQueue autoCreatedLeafQueue =
+ (AutoCreatedLeafQueue) cs.getQueue(USER0);
+ ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue(
+ PARENT_QUEUE);
+ assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
+ validateCapacities(autoCreatedLeafQueue);
+
+ ApplicationAttemptId appAttemptId = appsInC.get(0);
+
+ Priority priority = TestUtils.createMockPriority(1);
+ RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(
+ null);
+ ResourceRequest r1 = TestUtils.createResourceRequest(ResourceRequest.ANY,
+ 1 * GB, 1, true, priority, recordFactory);
+
+ cs.allocate(appAttemptId, Collections.<ResourceRequest>singletonList(r1),
+ Collections.<ContainerId>emptyList(), Collections.singletonList(host),
+ null, NULL_UPDATE_REQUESTS);
+
+ //And this will result in container assignment for app1
+ CapacityScheduler.schedule(cs);
+
+ //change state to draining
+ autoCreatedLeafQueue.stopQueue();
+
+ cs.killAllAppsInQueue(USER0);
+
+ mockRM.waitForState(appAttemptId, RMAppAttemptState.KILLED);
+
+ mockRM.waitForState(appAttemptId.getApplicationId(), RMAppState.KILLED);
+
+ //change state to stopped
+ autoCreatedLeafQueue.stopQueue();
+ assertEquals(QueueState.STOPPED,
+ autoCreatedLeafQueue.getQueueInfo().getQueueState());
+
+ cs.reinitialize(cs.getConf(), mockRM.getRMContext());
+
+ AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) cs.getQueue(
+ USER0);
+ validateCapacities(leafQueue);
+
+ } finally {
+ cleanupQueue(USER0);
+ }
+ }
+
+ @Test
+ public void testRefreshQueuesWithAutoCreatedLeafQueues() throws Exception {
+
+ MockRM newMockRM = setupSchedulerInstance();
+ try {
+ CapacityScheduler newCS =
+ (CapacityScheduler) newMockRM.getResourceScheduler();
+ CapacitySchedulerConfiguration conf = newCS.getConfiguration();
+
+ // Test add one auto created queue dynamically and manually modify
+ // capacity
+ ManagedParentQueue parentQueue = (ManagedParentQueue) newCS.getQueue("c");
+ AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS, "c1",
+ parentQueue);
+ newCS.addQueue(c1);
+ c1.setEntitlement(new QueueEntitlement(C1_CAPACITY / 100, 1f));
+
+ // Test add another auto created queue and use setEntitlement to modify
+ // capacity
+ AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS, "c2",
+ (ManagedParentQueue) newCS.getQueue("c"));
+ newCS.addQueue(c2);
+ newCS.setEntitlement("c2", new QueueEntitlement(C2_CAPACITY / 100, 1f));
+
+ // Verify all allocations match
+ checkQueueCapacities(newCS, C_CAPACITY, D_CAPACITY);
+
+ // Reinitialize and verify all dynamic queued survived
+
+ conf.setCapacity(A, 20f);
+ conf.setCapacity(B, 20f);
+ conf.setCapacity(C, 40f);
+ conf.setCapacity(D, 20f);
+ newCS.reinitialize(conf, newMockRM.getRMContext());
+
+ checkQueueCapacities(newCS, 40f, 20f);
+
+ //chnage parent template configs and reinitialize
+ conf.setAutoCreatedLeafQueueTemplateCapacity(C, 30.0f);
+ conf.setAutoCreatedLeafQueueTemplateMaxCapacity(C, 100.0f);
+ newCS.reinitialize(conf, newMockRM.getRMContext());
+
+ ManagedParentQueue c = (ManagedParentQueue) newCS.getQueue("c");
+ AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS, "c3", c);
+ newCS.addQueue(c3);
+
+ AbstractManagedParentQueue.AutoCreatedLeafQueueTemplate
+ leafQueueTemplate = parentQueue.getLeafQueueTemplate();
+ QueueCapacities cap = leafQueueTemplate.getQueueCapacities();
+ c3.setEntitlement(
+ new QueueEntitlement(cap.getCapacity(), cap.getMaximumCapacity()));
+ newCS.reinitialize(conf, newMockRM.getRMContext());
+
+ checkQueueCapacities(newCS, 40f, 20f);
+ } finally {
+ if (newMockRM != null) {
+ ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+ newMockRM.stop();
+ }
+ }
+ }
+
+ @Test
+ public void testConvertAutoCreateDisabledOnManagedParentQueueFails()
+ throws Exception {
+ CapacityScheduler newCS = new CapacityScheduler();
+ try {
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(newConf);
+
+ newConf.setAutoCreateChildQueueEnabled(C, false);
+
+ newCS.setConf(new YarnConfiguration());
+ newCS.setRMContext(mockRM.getRMContext());
+ newCS.init(cs.getConf());
+ newCS.start();
+
+ newCS.reinitialize(newConf,
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(newConf),
+ new NMTokenSecretManagerInRM(newConf),
+ new ClientToAMTokenSecretManagerInRM(), null));
+
+ } catch (IOException e) {
+ //expected exception
+ } finally {
+ newCS.stop();
+ }
+ }
+
+ @Test
+ public void testConvertLeafQueueToParentQueueWithAutoCreate()
+ throws Exception {
+ CapacityScheduler newCS = new CapacityScheduler();
+ try {
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(newConf);
+ newConf.setAutoCreatedLeafQueueTemplateCapacity(A1, A1_CAPACITY / 10);
+ newConf.setAutoCreateChildQueueEnabled(A1, true);
+
+ newCS.setConf(new YarnConfiguration());
+ newCS.setRMContext(mockRM.getRMContext());
+ newCS.init(cs.getConf());
+ newCS.start();
+
+ final LeafQueue a1Queue = (LeafQueue) newCS.getQueue("a1");
+ a1Queue.stopQueue();
+
+ newCS.reinitialize(newConf,
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(newConf),
+ new NMTokenSecretManagerInRM(newConf),
+ new ClientToAMTokenSecretManagerInRM(), null));
+
+ } finally {
+ newCS.stop();
+ }
+ }
+
+ @Test
+ public void testConvertFailsFromParentQueueToManagedParentQueue()
+ throws Exception {
+ CapacityScheduler newCS = new CapacityScheduler();
+ try {
+ CapacitySchedulerConfiguration newConf =
+ new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(newConf);
+ newConf.setAutoCreatedLeafQueueTemplateCapacity(A, A_CAPACITY / 10);
+ newConf.setAutoCreateChildQueueEnabled(A, true);
+
+ newCS.setConf(new YarnConfiguration());
+ newCS.setRMContext(mockRM.getRMContext());
+ newCS.init(cs.getConf());
+ newCS.start();
+
+ final ParentQueue a1Queue = (ParentQueue) newCS.getQueue("a");
+ a1Queue.stopQueue();
+
+ newCS.reinitialize(newConf,
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(newConf),
+ new NMTokenSecretManagerInRM(newConf),
+ new ClientToAMTokenSecretManagerInRM(), null));
+
+ fail("Expected exception while converting a parent queue to"
+ + " an auto create enabled parent queue");
+ } catch (IOException e) {
+ //expected exception
+ } finally {
+ newCS.stop();
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testAutoCreateLeafQueueFailsWithNoQueueMapping()
+ throws Exception {
+
+ final String INVALID_USER = "invalid_user";
+
+ // submit an app under a different queue name which does not exist
+ // and queue mapping does not exist for this user
+ RMApp app = mockRM.submitApp(GB, "app", INVALID_USER, null, INVALID_USER,
+ false);
+ mockRM.drainEvents();
+ mockRM.waitForState(app.getApplicationId(), RMAppState.FAILED);
+ assertEquals(RMAppState.FAILED, app.getState());
+ }
+
+ private void validateCapacities(AutoCreatedLeafQueue autoCreatedLeafQueue) {
+ assertEquals(autoCreatedLeafQueue.getCapacity(), 0.0f, EPSILON);
+ assertEquals(autoCreatedLeafQueue.getAbsoluteCapacity(), 0.0f, EPSILON);
+ assertEquals(autoCreatedLeafQueue.getMaximumCapacity(), 0.0f, EPSILON);
+ assertEquals(autoCreatedLeafQueue.getAbsoluteMaximumCapacity(), 0.0f,
+ EPSILON);
+ int maxAppsForAutoCreatedQueues = (int) (
+ CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS
+ * autoCreatedLeafQueue.getParent().getAbsoluteCapacity());
+ assertEquals(autoCreatedLeafQueue.getMaxApplicationsPerUser(),
+ maxAppsForAutoCreatedQueues);
+ assertEquals(autoCreatedLeafQueue.getMaxApplicationsPerUser(),
+ (int) (maxAppsForAutoCreatedQueues * (cs.getConfiguration()
+ .getUserLimitFactor(
+ autoCreatedLeafQueue.getParent().getQueuePath()))));
+ }
+
+ private void cleanupQueue(String queueName) throws YarnException {
+ AutoCreatedLeafQueue queue = (AutoCreatedLeafQueue) cs.getQueue(queueName);
+ if (queue != null) {
+ queue.setEntitlement(new QueueEntitlement(0.0f, 0.0f));
+ ((ManagedParentQueue) queue.getParent()).removeChildQueue(
+ queue.getQueueName());
+ cs.getCapacitySchedulerQueueManager().removeQueue(queue.getQueueName());
+ } else{
+ throw new YarnException("Queue does not exist " + queueName);
+ }
+ }
+
+ String getQueueMapping(String parentQueue, String leafQueue) {
+ return parentQueue + DOT + leafQueue;
+ }
+
+ @Test(timeout = 10000)
+ public void testQueueMappingValidationFailsWithInvalidParentQueueInMapping()
+ throws Exception {
+
+ MockRM newMockRM = setupSchedulerInstance();
+ try {
+ CapacityScheduler newCS =
+ (CapacityScheduler) newMockRM.getResourceScheduler();
+
+ //"a" is not auto create enabled
+
+ //dynamic queue mapping
+ try {
+ setupQueueMapping(newCS, CURRENT_USER_MAPPING, "a",
+ CURRENT_USER_MAPPING);
+ newCS.updatePlacementRules();
+ fail("Expected invalid parent queue mapping failure");
+
+ } catch (IOException e) {
+ //expected exception
+ assertTrue(e.getMessage().contains(
+ "invalid parent queue which does not have auto creation of leaf "
+ + "queues enabled ["
+ + "a" + "]"));
+ }
+
+ //"a" is not auto create enabled and app_user does not exist as a leaf
+ // queue
+ //static queue mapping
+ try {
+ setupQueueMapping(newCS, "app_user", "INVALID_PARENT_QUEUE",
+ "app_user");
+ newCS.updatePlacementRules();
+ fail("Expected invalid parent queue mapping failure");
+ } catch (IOException e) {
+ //expected exception
+ assertTrue(e.getMessage()
+ .contains("invalid parent queue [" + "INVALID_PARENT_QUEUE" + "]"));
+ }
+ } finally {
+ if (newMockRM != null) {
+ ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+ newMockRM.stop();
+ }
+ }
+ }
+
+ @Test(timeout = 10000)
+ public void testQueueMappingUpdatesFailsOnRemovalOfParentQueueInMapping()
+ throws Exception {
+
+ MockRM newMockRM = setupSchedulerInstance();
+
+ try {
+ CapacityScheduler newCS =
+ (CapacityScheduler) newMockRM.getResourceScheduler();
+
+ setupQueueMapping(newCS, CURRENT_USER_MAPPING, "c", CURRENT_USER_MAPPING);
+ newCS.updatePlacementRules();
+
+ try {
+ setupQueueMapping(newCS, CURRENT_USER_MAPPING, "",
+ CURRENT_USER_MAPPING);
+ newCS.updatePlacementRules();
+ fail("Expected invalid parent queue mapping failure");
+ } catch (IOException e) {
+ //expected exception
+ assertTrue(e.getMessage().contains("invalid parent queue []"));
+ }
+ } finally {
+ if (newMockRM != null) {
+ ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+ newMockRM.stop();
+ }
+ }
+ }
+
+ @Test
+ public void testParentQueueUpdateInQueueMappingFailsAfterAutoCreation()
+ throws Exception {
+
+ MockRM newMockRM = setupSchedulerInstance();
+ CapacityScheduler newCS =
+ (CapacityScheduler) newMockRM.getResourceScheduler();
+
+ try {
+ newMockRM.start();
+ newCS.start();
+
+ submitApp(newCS, USER0, USER0, PARENT_QUEUE);
+
+ assertNotNull(newCS.getQueue(USER0));
+
+ setupQueueMapping(newCS, USER0, "d", USER0);
+ newCS.updatePlacementRules();
+
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getDispatcher()).thenReturn(dispatcher);
+ newCS.setRMContext(rmContext);
+
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+ SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, USER0,
+ USER0, new ApplicationPlacementContext(USER0, "d"));
+ newCS.handle(addAppEvent);
+
+ RMAppEvent event = new RMAppEvent(appId, RMAppEventType.APP_REJECTED,
+ "error");
+ dispatcher.spyOnNextEvent(event, 10000);
+ } finally {
+ if (newMockRM != null) {
+ ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+ newMockRM.stop();
+ }
+ }
+ }
+
+ @Test
+ public void testAutoCreationFailsWhenParentCapacityExceeded()
+ throws IOException, SchedulerDynamicEditException {
+ MockRM newMockRM = setupSchedulerInstance();
+ CapacityScheduler newCS =
+ (CapacityScheduler) newMockRM.getResourceScheduler();
+
+ try {
+ CapacitySchedulerConfiguration conf = newCS.getConfiguration();
+ conf.setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(C,
+ true);
+
+ newCS.reinitialize(conf, newMockRM.getRMContext());
+
+ // Test add one auto created queue dynamically and manually modify
+ // capacity
+ ManagedParentQueue parentQueue = (ManagedParentQueue) newCS.getQueue("c");
+ AutoCreatedLeafQueue c1 = new AutoCreatedLeafQueue(newCS, "c1",
+ parentQueue);
+ newCS.addQueue(c1);
+ c1.setEntitlement(new QueueEntitlement(0.5f, 1f));
+
+ AutoCreatedLeafQueue c2 = new AutoCreatedLeafQueue(newCS, "c2",
+ parentQueue);
+ newCS.addQueue(c2);
+ c2.setEntitlement(new QueueEntitlement(0.5f, 1f));
+
+ try {
+ AutoCreatedLeafQueue c3 = new AutoCreatedLeafQueue(newCS, "c3",
+ parentQueue);
+ newCS.addQueue(c3);
+ fail("Expected exception for auto queue creation failure");
+ } catch (SchedulerDynamicEditException e) {
+ //expected exception
+ }
+ } finally {
+ if (newMockRM != null) {
+ ((CapacityScheduler) newMockRM.getResourceScheduler()).stop();
+ newMockRM.stop();
+ }
+ }
+ }
+
+ private List<UserGroupMappingPlacementRule.QueueMapping> setupQueueMapping(
+ CapacityScheduler newCS, String user, String parentQueue, String queue) {
+ List<UserGroupMappingPlacementRule.QueueMapping> queueMappings =
+ new ArrayList<>();
+ queueMappings.add(new UserGroupMappingPlacementRule.QueueMapping(
+ UserGroupMappingPlacementRule.QueueMapping.MappingType.USER, user,
+ getQueueMapping(parentQueue, queue)));
+ newCS.getConfiguration().setQueueMappings(queueMappings);
+ return queueMappings;
+ }
+
+ private MockRM setupSchedulerInstance() {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+
+ List<String> queuePlacementRules = new ArrayList<String>();
+ queuePlacementRules.add(YarnConfiguration.USER_GROUP_PLACEMENT_RULE);
+ conf.setQueuePlacementRules(queuePlacementRules);
+
+ setupQueueMappings(conf);
+
+ MockRM newMockRM = new MockRM(conf);
+ return newMockRM;
+ }
+
+ void checkQueueCapacities(CapacityScheduler newCS, float capacityC,
+ float capacityD) {
+ CSQueue rootQueue = newCS.getRootQueue();
+ CSQueue queueC = tcs.findQueue(rootQueue, C);
+ CSQueue queueD = tcs.findQueue(rootQueue, D);
+ CSQueue queueC1 = tcs.findQueue(queueC, C1);
+ CSQueue queueC2 = tcs.findQueue(queueC, C2);
+ CSQueue queueC3 = tcs.findQueue(queueC, C3);
+
+ float capC = capacityC / 100.0f;
+ float capD = capacityD / 100.0f;
+
+ tcs.checkQueueCapacity(queueC, capC, capC, 1.0f, 1.0f);
+ tcs.checkQueueCapacity(queueD, capD, capD, 1.0f, 1.0f);
+ tcs.checkQueueCapacity(queueC1, C1_CAPACITY / 100.0f,
+ (C1_CAPACITY / 100.0f) * capC, 1.0f, 1.0f);
+ tcs.checkQueueCapacity(queueC2, C2_CAPACITY / 100.0f,
+ (C2_CAPACITY / 100.0f) * capC, 1.0f, 1.0f);
+
+ if (queueC3 != null) {
+ ManagedParentQueue parentQueue = (ManagedParentQueue) queueC;
+ QueueCapacities cap =
+ parentQueue.getLeafQueueTemplate().getQueueCapacities();
+ tcs.checkQueueCapacity(queueC3, cap.getCapacity(),
+ (cap.getCapacity()) * capC, 1.0f, 1.0f);
+ }
+ }
+
+ ApplicationAttemptId submitApp(CapacityScheduler newCS, String user,
+ String queue, String parentQueue) {
+ ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
+ SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, queue, user,
+ new ApplicationPlacementContext(queue, parentQueue));
+ ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
+ appId, 1);
+ SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(
+ appAttemptId, false);
+ newCS.handle(addAppEvent);
+ newCS.handle(addAttemptEvent);
+ return appAttemptId;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/19] hadoop git commit: HDFS-12814. Add blockId when warning slow
mirror/disk in BlockReceiver. Contributed by Jiandan Yang.
Posted by as...@apache.org.
HDFS-12814. Add blockId when warning slow mirror/disk in BlockReceiver. Contributed by Jiandan Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/462e25a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/462e25a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/462e25a3
Branch: refs/heads/YARN-6592
Commit: 462e25a3b264e1148d0cbca00db7f10d43a0555f
Parents: 09a1342
Author: Weiwei Yang <ww...@apache.org>
Authored: Thu Nov 16 16:19:53 2017 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Thu Nov 16 16:19:53 2017 +0800
----------------------------------------------------------------------
.../hadoop/hdfs/server/datanode/BlockReceiver.java | 16 +++++++++++-----
1 file changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462e25a3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 8d91f04..c052d52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -434,7 +434,8 @@ class BlockReceiver implements Closeable {
if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow flushOrSync took " + duration + "ms (threshold="
+ datanodeSlowLogThresholdMs + "ms), isSync:" + isSync + ", flushTotalNanos="
- + flushTotalNanos + "ns, volume=" + getVolumeBaseUri());
+ + flushTotalNanos + "ns, volume=" + getVolumeBaseUri()
+ + ", blockId=" + replicaInfo.getBlockId());
}
}
@@ -591,7 +592,8 @@ class BlockReceiver implements Closeable {
if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow BlockReceiver write packet to mirror took " + duration
+ "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), "
- + "downstream DNs=" + Arrays.toString(downstreamDNs));
+ + "downstream DNs=" + Arrays.toString(downstreamDNs)
+ + ", blockId=" + replicaInfo.getBlockId());
}
} catch (IOException e) {
handleMirrorOutError(e);
@@ -725,7 +727,8 @@ class BlockReceiver implements Closeable {
if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow BlockReceiver write data to disk cost:" + duration
+ "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), "
- + "volume=" + getVolumeBaseUri());
+ + "volume=" + getVolumeBaseUri()
+ + ", blockId=" + replicaInfo.getBlockId());
}
if (duration > maxWriteToDiskMs) {
@@ -917,7 +920,8 @@ class BlockReceiver implements Closeable {
if (duration > datanodeSlowLogThresholdMs && LOG.isWarnEnabled()) {
LOG.warn("Slow manageWriterOsCache took " + duration
+ "ms (threshold=" + datanodeSlowLogThresholdMs
- + "ms), volume=" + getVolumeBaseUri());
+ + "ms), volume=" + getVolumeBaseUri()
+ + ", blockId=" + replicaInfo.getBlockId());
}
}
} catch (Throwable t) {
@@ -1629,7 +1633,9 @@ class BlockReceiver implements Closeable {
if (duration > datanodeSlowLogThresholdMs) {
LOG.warn("Slow PacketResponder send ack to upstream took " + duration
+ "ms (threshold=" + datanodeSlowLogThresholdMs + "ms), " + myString
- + ", replyAck=" + replyAck);
+ + ", replyAck=" + replyAck
+ + ", downstream DNs=" + Arrays.toString(downstreamDNs)
+ + ", blockId=" + replicaInfo.getBlockId());
} else if (LOG.isDebugEnabled()) {
LOG.debug(myString + ", replyAck=" + replyAck);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/19] hadoop git commit: HADOOP-14982. Clients using
FailoverOnNetworkExceptionRetry can go into a loop if they're used without
authenticating with kerberos in HA env (pbacsko via rkanter)
Posted by as...@apache.org.
HADOOP-14982. Clients using FailoverOnNetworkExceptionRetry can go into a loop if they're used without authenticating with kerberos in HA env (pbacsko via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2efaf01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2efaf01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2efaf01
Branch: refs/heads/YARN-6592
Commit: f2efaf013f7577948061abbb49c6d17c375e92cc
Parents: 6bf2c30
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Nov 16 11:11:19 2017 -0800
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Nov 16 11:11:19 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/io/retry/RetryPolicies.java | 22 +++++++++++++++++++-
.../apache/hadoop/io/retry/TestRetryProxy.java | 22 ++++++++++++++++++++
.../io/retry/UnreliableImplementation.java | 10 +++++++++
.../hadoop/io/retry/UnreliableInterface.java | 6 +++++-
4 files changed, 58 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2efaf01/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
index fa0cb6e..adf23c0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryPolicies.java
@@ -32,11 +32,14 @@ import java.util.Map.Entry;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.TimeUnit;
+import javax.security.sasl.SaslException;
+
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RetriableException;
import org.apache.hadoop.ipc.StandbyException;
import org.apache.hadoop.net.ConnectTimeoutException;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.ietf.jgss.GSSException;
import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
@@ -663,6 +666,11 @@ public class RetryPolicies {
+ retries + ") exceeded maximum allowed (" + maxRetries + ")");
}
+ if (isSaslFailure(e)) {
+ return new RetryAction(RetryAction.RetryDecision.FAIL, 0,
+ "SASL failure");
+ }
+
if (e instanceof ConnectException ||
e instanceof EOFException ||
e instanceof NoRouteToHostException ||
@@ -716,7 +724,7 @@ public class RetryPolicies {
private static long calculateExponentialTime(long time, int retries) {
return calculateExponentialTime(time, retries, Long.MAX_VALUE);
}
-
+
private static boolean isWrappedStandbyException(Exception e) {
if (!(e instanceof RemoteException)) {
return false;
@@ -725,6 +733,18 @@ public class RetryPolicies {
StandbyException.class);
return unwrapped instanceof StandbyException;
}
+
+ private static boolean isSaslFailure(Exception e) {
+ Throwable current = e;
+ do {
+ if (current instanceof SaslException) {
+ return true;
+ }
+ current = current.getCause();
+ } while (current != null);
+
+ return false;
+ }
static RetriableException getWrappedRetriableException(Exception e) {
if (!(e instanceof RemoteException)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2efaf01/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
index 649af89..1accb0a0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/TestRetryProxy.java
@@ -39,6 +39,8 @@ import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import javax.security.sasl.SaslException;
+
import static org.apache.hadoop.io.retry.RetryPolicies.*;
import static org.junit.Assert.*;
import static org.mockito.Matchers.any;
@@ -326,4 +328,24 @@ public class TestRetryProxy {
assertEquals(InterruptedException.class, e.getCause().getClass());
assertEquals("sleep interrupted", e.getCause().getMessage());
}
+
+ @Test
+ public void testNoRetryOnSaslError() throws Exception {
+ RetryPolicy policy = mock(RetryPolicy.class);
+ RetryPolicy realPolicy = RetryPolicies.failoverOnNetworkException(5);
+ setupMockPolicy(policy, realPolicy);
+
+ UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(
+ UnreliableInterface.class, unreliableImpl, policy);
+
+ try {
+ unreliable.failsWithSASLExceptionTenTimes();
+ fail("Should fail");
+ } catch (SaslException e) {
+ // expected
+ verify(policy, times(1)).shouldRetry(any(Exception.class), anyInt(),
+ anyInt(), anyBoolean());
+ assertEquals(RetryDecision.FAIL, caughtRetryAction.action);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2efaf01/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
index 7ce8abc..465dc6f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableImplementation.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.io.retry;
import java.io.IOException;
+import javax.security.sasl.SaslException;
+
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
@@ -29,6 +31,7 @@ class UnreliableImplementation implements UnreliableInterface {
failsOnceIOExceptionInvocationCount,
failsOnceRemoteExceptionInvocationCount,
failsTenTimesInvocationCount,
+ failsWithSASLExceptionTenTimesInvocationCount,
succeedsOnceThenFailsCount,
succeedsOnceThenFailsIdempotentCount,
succeedsTenTimesThenFailsCount;
@@ -114,6 +117,13 @@ class UnreliableImplementation implements UnreliableInterface {
}
@Override
+ public void failsWithSASLExceptionTenTimes() throws SaslException {
+ if (failsWithSASLExceptionTenTimesInvocationCount ++ < 10) {
+ throw new SaslException();
+ }
+ }
+
+ @Override
public String succeedsOnceThenFailsReturningString()
throws UnreliableException, IOException, StandbyException {
if (succeedsOnceThenFailsCount++ < 1) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2efaf01/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
index 6c9c153..d334542 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/retry/UnreliableInterface.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.io.retry;
import java.io.IOException;
+import javax.security.sasl.SaslException;
+
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.StandbyException;
@@ -61,7 +63,9 @@ public interface UnreliableInterface {
boolean failsOnceThenSucceedsWithReturnValue() throws UnreliableException;
void failsTenTimesThenSucceeds() throws UnreliableException;
-
+
+ void failsWithSASLExceptionTenTimes() throws SaslException;
+
public String succeedsOnceThenFailsReturningString()
throws UnreliableException, StandbyException, IOException;
@Idempotent
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/19] hadoop git commit: YARN-7419. CapacityScheduler: Allow auto
leaf queue creation after queue mapping. (Suma Shivaprasad via wangda)
Posted by as...@apache.org.
YARN-7419. CapacityScheduler: Allow auto leaf queue creation after queue mapping. (Suma Shivaprasad via wangda)
Change-Id: Ia1704bb8cb5070e5b180b5a85787d7b9ca57ebc6
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0987a7b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0987a7b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0987a7b8
Branch: refs/heads/YARN-6592
Commit: 0987a7b8cbbbb2c1e4c2095821d98a7db19644df
Parents: f2efaf0
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Nov 16 11:22:48 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Nov 16 11:25:52 2017 -0800
----------------------------------------------------------------------
.../server/resourcemanager/RMAppManager.java | 7 +-
.../placement/ApplicationPlacementContext.java | 52 ++
.../placement/PlacementManager.java | 34 +-
.../placement/PlacementRule.java | 7 +-
.../UserGroupMappingPlacementRule.java | 284 ++++++-
.../server/resourcemanager/rmapp/RMAppImpl.java | 87 +-
.../scheduler/capacity/AbstractCSQueue.java | 2 +-
.../capacity/AbstractManagedParentQueue.java | 196 +++--
.../capacity/AutoCreatedLeafQueue.java | 27 +-
.../scheduler/capacity/CapacityScheduler.java | 157 +++-
.../CapacitySchedulerConfiguration.java | 153 ++++
.../capacity/CapacitySchedulerQueueManager.java | 103 ++-
.../scheduler/capacity/ManagedParentQueue.java | 158 ++++
.../scheduler/capacity/ParentQueue.java | 13 -
.../scheduler/capacity/PlanQueue.java | 25 +-
.../scheduler/event/AppAddedSchedulerEvent.java | 37 +-
.../server/resourcemanager/TestAppManager.java | 29 +-
.../TestUserGroupMappingPlacementRule.java | 14 +-
.../scheduler/TestSchedulerUtils.java | 1 +
.../capacity/TestCapacityScheduler.java | 6 +-
.../TestCapacitySchedulerAutoQueueCreation.java | 794 +++++++++++++++++++
21 files changed, 1921 insertions(+), 265 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index d042590..5e82f40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -360,13 +360,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
private RMAppImpl createAndPopulateNewRMApp(
ApplicationSubmissionContext submissionContext, long submitTime,
String user, boolean isRecovery, long startTime) throws YarnException {
+
if (!isRecovery) {
- // Do queue mapping
- if (rmContext.getQueuePlacementManager() != null) {
- // We only do queue mapping when it's a new application
- rmContext.getQueuePlacementManager().placeApplication(
- submissionContext, user);
- }
// fail the submission if configured application timeout value is invalid
RMServerUtils.validateApplicationTimeouts(
submissionContext.getApplicationTimeouts());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
new file mode 100644
index 0000000..f2f92b8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/ApplicationPlacementContext.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.placement;
+
+/**
+ * Each placement rule when it successfully places an application onto a queue
+ * returns a PlacementRuleContext which encapsulates the queue the
+ * application was mapped to and any parent queue for the queue (if configured)
+ */
+public class ApplicationPlacementContext {
+
+ private String queue;
+
+ private String parentQueue;
+
+ public ApplicationPlacementContext(String queue) {
+ this(queue,null);
+ }
+
+ public ApplicationPlacementContext(String queue, String parentQueue) {
+ this.queue = queue;
+ this.parentQueue = parentQueue;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public String getParentQueue() {
+ return parentQueue;
+ }
+
+ public boolean hasParentQueue() {
+ return parentQueue != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
index 43a4deb..c006738 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementManager.java
@@ -23,7 +23,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -53,36 +52,33 @@ public class PlacementManager {
}
}
- public void placeApplication(ApplicationSubmissionContext asc, String user)
- throws YarnException {
+ public ApplicationPlacementContext placeApplication(
+ ApplicationSubmissionContext asc, String user) throws YarnException {
+
try {
readLock.lock();
+
if (null == rules || rules.isEmpty()) {
- return;
+ return null;
}
-
- String newQueueName = null;
+
+ ApplicationPlacementContext placement = null;
for (PlacementRule rule : rules) {
- newQueueName = rule.getQueueForApp(asc, user);
- if (newQueueName != null) {
+ placement = rule.getPlacementForApp(asc, user);
+ if (placement != null) {
break;
}
}
-
+
// Failed to get where to place application
- if (null == newQueueName && null == asc.getQueue()) {
- String msg = "Failed to get where to place application="
- + asc.getApplicationId();
+ if (null == placement && null == asc.getQueue()) {
+ String msg = "Failed to get where to place application=" + asc
+ .getApplicationId();
LOG.error(msg);
throw new YarnException(msg);
}
-
- // Set it to ApplicationSubmissionContext
- if (!StringUtils.equals(asc.getQueue(), newQueueName)) {
- LOG.info("Placed application=" + asc.getApplicationId() + " to queue="
- + newQueueName + ", original queue=" + asc.getQueue());
- asc.setQueue(newQueueName);
- }
+
+ return placement;
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
index 47dc48a..a9d5e33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/PlacementRule.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
public abstract class PlacementRule {
+
public String getName() {
return this.getClass().getName();
}
@@ -50,6 +51,6 @@ public abstract class PlacementRule {
* in the {@link PlacementManager} will take care
* </p>
*/
- public abstract String getQueueForApp(ApplicationSubmissionContext asc,
- String user) throws YarnException;
-}
+ public abstract ApplicationPlacementContext getPlacementForApp(
+ ApplicationSubmissionContext asc, String user) throws YarnException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
index d617d16..9901f4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/placement/UserGroupMappingPlacementRule.java
@@ -19,8 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.placement;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -32,6 +34,15 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping.MappingType;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AutoCreatedLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerQueueManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ManagedParentQueue;
+
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.DOT;
public class UserGroupMappingPlacementRule extends PlacementRule {
private static final Log LOG = LogFactory
@@ -66,17 +77,41 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
MappingType type;
String source;
String queue;
+ String parentQueue;
+
+ public final static String DELIMITER = ":";
public QueueMapping(MappingType type, String source, String queue) {
this.type = type;
this.source = source;
this.queue = queue;
+ this.parentQueue = null;
}
-
+
+ public QueueMapping(MappingType type, String source,
+ String queue, String parentQueue) {
+ this.type = type;
+ this.source = source;
+ this.queue = queue;
+ this.parentQueue = parentQueue;
+ }
+
public String getQueue() {
return queue;
}
-
+
+ public String getParentQueue() {
+ return parentQueue;
+ }
+
+ public MappingType getType() {
+ return type;
+ }
+
+ public String getSource() {
+ return source;
+ }
+
@Override
public int hashCode() {
return super.hashCode();
@@ -93,6 +128,13 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
return false;
}
}
+
+ public String toString() {
+ return type.toString() + DELIMITER + source + DELIMITER +
+ (parentQueue != null ?
+ parentQueue + "." + queue :
+ queue);
+ }
}
public UserGroupMappingPlacementRule(boolean overrideWithQueueMappings,
@@ -102,26 +144,27 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
this.groups = groups;
}
- private String getMappedQueue(String user) throws IOException {
+ private ApplicationPlacementContext getPlacementForUser(String user)
+ throws IOException {
for (QueueMapping mapping : mappings) {
if (mapping.type == MappingType.USER) {
if (mapping.source.equals(CURRENT_USER_MAPPING)) {
if (mapping.queue.equals(CURRENT_USER_MAPPING)) {
- return user;
+ return getPlacementContext(mapping, user);
} else if (mapping.queue.equals(PRIMARY_GROUP_MAPPING)) {
- return groups.getGroups(user).get(0);
+ return getPlacementContext(mapping, groups.getGroups(user).get(0));
} else {
- return mapping.queue;
+ return getPlacementContext(mapping);
}
}
if (user.equals(mapping.source)) {
- return mapping.queue;
+ return getPlacementContext(mapping);
}
}
if (mapping.type == MappingType.GROUP) {
for (String userGroups : groups.getGroups(user)) {
if (userGroups.equals(mapping.source)) {
- return mapping.queue;
+ return getPlacementContext(mapping);
}
}
}
@@ -130,13 +173,14 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
}
@Override
- public String getQueueForApp(ApplicationSubmissionContext asc, String user)
+ public ApplicationPlacementContext getPlacementForApp(
+ ApplicationSubmissionContext asc, String user)
throws YarnException {
String queueName = asc.getQueue();
ApplicationId applicationId = asc.getApplicationId();
if (mappings != null && mappings.size() > 0) {
try {
- String mappedQueue = getMappedQueue(user);
+ ApplicationPlacementContext mappedQueue = getPlacementForUser(user);
if (mappedQueue != null) {
// We have a mapping, should we use it?
if (queueName.equals(YarnConfiguration.DEFAULT_QUEUE_NAME)
@@ -153,10 +197,224 @@ public class UserGroupMappingPlacementRule extends PlacementRule {
throw new YarnException(message);
}
}
-
- return queueName;
+ return null;
}
-
+
+ private ApplicationPlacementContext getPlacementContext(
+ QueueMapping mapping) {
+ return getPlacementContext(mapping, mapping.getQueue());
+ }
+
+ private ApplicationPlacementContext getPlacementContext(QueueMapping mapping,
+ String leafQueueName) {
+ if (!StringUtils.isEmpty(mapping.parentQueue)) {
+ return new ApplicationPlacementContext(leafQueueName,
+ mapping.getParentQueue());
+ } else{
+ return new ApplicationPlacementContext(leafQueueName);
+ }
+ }
+
+ @VisibleForTesting
+ public static UserGroupMappingPlacementRule get(
+ CapacitySchedulerContext schedulerContext) throws IOException {
+ CapacitySchedulerConfiguration conf = schedulerContext.getConfiguration();
+ boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
+ LOG.info(
+ "Initialized queue mappings, override: " + overrideWithQueueMappings);
+
+ List<QueueMapping> queueMappings = conf.getQueueMappings();
+
+ // Get new user/group mappings
+ List<QueueMapping> newMappings = new ArrayList<>();
+
+ CapacitySchedulerQueueManager queueManager =
+ schedulerContext.getCapacitySchedulerQueueManager();
+
+ // check if mappings refer to valid queues
+ for (QueueMapping mapping : queueMappings) {
+
+ QueuePath queuePath = extractQueuePath(mapping.getQueue());
+ if (isStaticQueueMapping(mapping)) {
+ //Try getting queue by its leaf queue name
+ // without splitting into parent/leaf queues
+ CSQueue queue = queueManager.getQueue(mapping.getQueue());
+ if (ifQueueDoesNotExist(queue)) {
+ //Try getting the queue by extracting leaf and parent queue names
+ //Assuming its a potential auto created leaf queue
+ queue = queueManager.getQueue(queuePath.getLeafQueue());
+
+ if (ifQueueDoesNotExist(queue)) {
+ //if leaf queue does not exist,
+ // this could be a potential auto created leaf queue
+ //validate if parent queue is specified,
+ // then it should exist and
+ // be an instance of AutoCreateEnabledParentQueue
+ QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
+ queueManager, mapping, queuePath);
+ if (newMapping == null) {
+ throw new IOException(
+ "mapping contains invalid or non-leaf queue " + mapping
+ .getQueue());
+ }
+ newMappings.add(newMapping);
+ } else{
+ QueueMapping newMapping = validateAndGetQueueMapping(queueManager,
+ queue, mapping, queuePath);
+ newMappings.add(newMapping);
+ }
+ } else{
+ // if queue exists, validate
+ // if its an instance of leaf queue
+ // if its an instance of auto created leaf queue,
+ // then extract parent queue name and update queue mapping
+ QueueMapping newMapping = validateAndGetQueueMapping(queueManager,
+ queue, mapping, queuePath);
+ newMappings.add(newMapping);
+ }
+ } else{
+ //If it is a dynamic queue mapping,
+ // we can safely assume leaf queue name does not have '.' in it
+ // validate
+ // if parent queue is specified, then
+ // parent queue exists and an instance of AutoCreateEnabledParentQueue
+ //
+ QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
+ queueManager, mapping, queuePath);
+ if (newMapping != null) {
+ newMappings.add(newMapping);
+ } else{
+ newMappings.add(mapping);
+ }
+ }
+ }
+
+ // initialize groups if mappings are present
+ if (newMappings.size() > 0) {
+ Groups groups = new Groups(conf);
+ return new UserGroupMappingPlacementRule(overrideWithQueueMappings,
+ newMappings, groups);
+ }
+
+ return null;
+ }
+
+ private static QueueMapping validateAndGetQueueMapping(
+ CapacitySchedulerQueueManager queueManager, CSQueue queue,
+ QueueMapping mapping, QueuePath queuePath) throws IOException {
+ if (!(queue instanceof LeafQueue)) {
+ throw new IOException(
+ "mapping contains invalid or non-leaf queue : " + mapping.getQueue());
+ }
+
+ if (queue instanceof AutoCreatedLeafQueue && queue
+ .getParent() instanceof ManagedParentQueue) {
+
+ QueueMapping newMapping = validateAndGetAutoCreatedQueueMapping(
+ queueManager, mapping, queuePath);
+ if (newMapping == null) {
+ throw new IOException(
+ "mapping contains invalid or non-leaf queue " + mapping.getQueue());
+ }
+ return newMapping;
+ }
+ return mapping;
+ }
+
+ private static boolean ifQueueDoesNotExist(CSQueue queue) {
+ return queue == null;
+ }
+
+ private static QueueMapping validateAndGetAutoCreatedQueueMapping(
+ CapacitySchedulerQueueManager queueManager, QueueMapping mapping,
+ QueuePath queuePath) throws IOException {
+ if (queuePath.hasParentQueue()) {
+ //if parent queue is specified,
+ // then it should exist and be an instance of ManagedParentQueue
+ validateParentQueue(queueManager.getQueue(queuePath.getParentQueue()),
+ queuePath.getParentQueue(), queuePath.getLeafQueue());
+ return new QueueMapping(mapping.getType(), mapping.getSource(),
+ queuePath.getLeafQueue(), queuePath.getParentQueue());
+ }
+
+ return null;
+ }
+
+ private static boolean isStaticQueueMapping(QueueMapping mapping) {
+ return !mapping.getQueue().contains(
+ UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mapping
+ .getQueue().contains(
+ UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING);
+ }
+
+ private static class QueuePath {
+
+ public String parentQueue;
+ public String leafQueue;
+
+ public QueuePath(final String leafQueue) {
+ this.leafQueue = leafQueue;
+ }
+
+ public QueuePath(final String parentQueue, final String leafQueue) {
+ this.parentQueue = parentQueue;
+ this.leafQueue = leafQueue;
+ }
+
+ public String getParentQueue() {
+ return parentQueue;
+ }
+
+ public String getLeafQueue() {
+ return leafQueue;
+ }
+
+ public boolean hasParentQueue() {
+ return parentQueue != null;
+ }
+
+ @Override
+ public String toString() {
+ return parentQueue + DOT + leafQueue;
+ }
+ }
+
+ private static QueuePath extractQueuePath(String queueName)
+ throws IOException {
+ int parentQueueNameEndIndex = queueName.lastIndexOf(DOT);
+
+ if (parentQueueNameEndIndex > -1) {
+ final String parentQueue = queueName.substring(0, parentQueueNameEndIndex)
+ .trim();
+ final String leafQueue = queueName.substring(parentQueueNameEndIndex + 1)
+ .trim();
+ return new QueuePath(parentQueue, leafQueue);
+ }
+
+ return new QueuePath(queueName);
+ }
+
+ private static void validateParentQueue(CSQueue parentQueue,
+ String parentQueueName, String leafQueueName) throws IOException {
+ if (parentQueue == null) {
+ throw new IOException(
+ "mapping contains invalid or non-leaf queue [" + leafQueueName
+ + "] and invalid parent queue [" + parentQueueName + "]");
+ } else if (!(parentQueue instanceof ManagedParentQueue)) {
+ throw new IOException("mapping contains leaf queue [" + leafQueueName
+ + "] and invalid parent queue which "
+ + "does not have auto creation of leaf queues enabled ["
+ + parentQueueName + "]");
+ } else if (!parentQueue.getQueueName().equals(parentQueueName)) {
+ throw new IOException(
+ "mapping contains invalid or non-leaf queue [" + leafQueueName
+ + "] and invalid parent queue "
+ + "which does not match existing leaf queue's parent : ["
+ + parentQueueName + "] does not match [ " + parentQueue
+ .getQueueName() + "]");
+ }
+ }
+
@VisibleForTesting
public List<QueueMapping> getQueueMappings() {
return mappings;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index ae5f6b4..85d355f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -68,6 +68,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
@@ -83,6 +84,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.blacklist.BlacklistManager;
import org.apache.hadoop.yarn.server.resourcemanager.blacklist.DisabledBlacklistManager;
import org.apache.hadoop.yarn.server.resourcemanager.blacklist.SimpleBlacklistManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
@@ -158,6 +161,8 @@ public class RMAppImpl implements RMApp, Recoverable {
private boolean isNumAttemptsBeyondThreshold = false;
+
+
// Mutable fields
private long startTime;
private long finishTime = 0;
@@ -1073,38 +1078,51 @@ public class RMAppImpl implements RMApp, Recoverable {
app.getUser(),
BuilderUtils.parseTokensConf(app.submissionContext));
} catch (Exception e) {
- String msg = "Failed to fetch user credentials from application:"
- + e.getMessage();
+ String msg = "Failed to fetch user credentials from application:" + e
+ .getMessage();
app.diagnostics.append(msg);
LOG.error(msg, e);
}
}
- for (Map.Entry<ApplicationTimeoutType, Long> timeout :
- app.applicationTimeouts.entrySet()) {
+ for (Map.Entry<ApplicationTimeoutType, Long> timeout : app.applicationTimeouts
+ .entrySet()) {
app.rmContext.getRMAppLifetimeMonitor().registerApp(app.applicationId,
timeout.getKey(), timeout.getValue());
if (LOG.isDebugEnabled()) {
long remainingTime = timeout.getValue() - app.systemClock.getTime();
LOG.debug("Application " + app.applicationId
+ " is registered for timeout monitor, type=" + timeout.getKey()
- + " remaining timeout="
- + (remainingTime > 0 ? remainingTime / 1000 : 0) + " seconds");
+ + " remaining timeout=" + (remainingTime > 0 ?
+ remainingTime / 1000 :
+ 0) + " seconds");
}
}
+ ApplicationPlacementContext placementContext = null;
+ try {
+ placementContext = placeApplication(app.rmContext,
+ app.submissionContext, app.user);
+ } catch (Exception e) {
+ String msg = "Failed to place application to queue :" + e.getMessage();
+ app.diagnostics.append(msg);
+ LOG.error(msg, e);
+ }
+
// No existent attempts means the attempt associated with this app was not
// started or started but not yet saved.
if (app.attempts.isEmpty()) {
- app.scheduler.handle(new AppAddedSchedulerEvent(app.user,
- app.submissionContext, false, app.applicationPriority));
+ app.scheduler.handle(
+ new AppAddedSchedulerEvent(app.user, app.submissionContext, false,
+ app.applicationPriority, placementContext));
return RMAppState.SUBMITTED;
}
// Add application to scheduler synchronously to guarantee scheduler
// knows applications before AM or NM re-registers.
- app.scheduler.handle(new AppAddedSchedulerEvent(app.user,
- app.submissionContext, true, app.applicationPriority));
+ app.scheduler.handle(
+ new AppAddedSchedulerEvent(app.user, app.submissionContext, true,
+ app.applicationPriority, placementContext));
// recover attempts
app.recoverAppAttempts();
@@ -1120,8 +1138,20 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppTransition {
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
- app.handler.handle(new AppAddedSchedulerEvent(app.user,
- app.submissionContext, false, app.applicationPriority));
+ ApplicationPlacementContext placementContext = null;
+ try {
+ placementContext = placeApplication(app.rmContext,
+ app.submissionContext, app.user);
+ replaceQueueFromPlacementContext(placementContext,
+ app.submissionContext);
+ } catch (YarnException e) {
+ String msg = "Failed to place application to queue :" + e.getMessage();
+ app.diagnostics.append(msg);
+ LOG.error(msg, e);
+ }
+ app.handler.handle(
+ new AppAddedSchedulerEvent(app.user, app.submissionContext, false,
+ app.applicationPriority, placementContext));
// send the ATS create Event
app.sendATSCreateEvent();
}
@@ -2013,4 +2043,37 @@ public class RMAppImpl implements RMApp, Recoverable {
this.submissionContext.setAMContainerSpec(null);
this.submissionContext.setLogAggregationContext(null);
}
+
+ @VisibleForTesting
+ static ApplicationPlacementContext placeApplication(RMContext rmContext,
+ ApplicationSubmissionContext context, String user) throws YarnException {
+
+ ApplicationPlacementContext placementContext = null;
+ PlacementManager placementManager = rmContext.getQueuePlacementManager();
+
+ if (placementManager != null) {
+ placementContext = placementManager.placeApplication(context, user);
+ } else{
+ LOG.error(
+ "Queue Placement Manager is null. Cannot place application :" + " "
+ + context.getApplicationId() + " to queue ");
+ }
+
+ return placementContext;
+ }
+
+ static void replaceQueueFromPlacementContext(
+ ApplicationPlacementContext placementContext,
+ ApplicationSubmissionContext context) {
+ // Set it to ApplicationSubmissionContext
+ //apply queue mapping only to new application submissions
+ if (placementContext != null && !StringUtils.equals(context.getQueue(),
+ placementContext.getQueue())) {
+ LOG.info("Placed application=" + context.getApplicationId() + " to queue="
+ + placementContext.getQueue() + ", original queue=" + context
+ .getQueue());
+ context.setQueue(placementContext.getQueue());
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 183cb36..74c85ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -78,7 +78,7 @@ public abstract class AbstractCSQueue implements CSQueue {
final String queueName;
private final String queuePath;
volatile int numContainers;
-
+
final Resource minimumAllocation;
volatile Resource maximumAllocation;
private volatile QueueState state = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
index b3d1b47..46f5cf1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractManagedParentQueue.java
@@ -35,31 +35,13 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
private static final Logger LOG = LoggerFactory.getLogger(
AbstractManagedParentQueue.class);
- private int maxAppsForAutoCreatedQueues;
- private int maxAppsPerUserForAutoCreatedQueues;
- private int userLimit;
- private float userLimitFactor;
+ protected AutoCreatedLeafQueueTemplate leafQueueTemplate;
public AbstractManagedParentQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
super.setupQueueConfigs(csContext.getClusterResource());
- initializeLeafQueueConfigs();
-
- StringBuffer queueInfo = new StringBuffer();
- queueInfo.append("Created Managed Parent Queue: ").append(queueName)
- .append("\nof type : [" + getClass())
- .append("]\nwith capacity: [")
- .append(super.getCapacity()).append("]\nwith max capacity: [")
- .append(super.getMaximumCapacity()).append("\nwith max apps: [")
- .append(getMaxApplicationsForAutoCreatedQueues())
- .append("]\nwith max apps per user: [")
- .append(getMaxApplicationsPerUserForAutoCreatedQueues())
- .append("]\nwith user limit: [").append(getUserLimit())
- .append("]\nwith user limit factor: [")
- .append(getUserLimitFactor()).append("].");
- LOG.info(queueInfo.toString());
}
@Override
@@ -71,8 +53,6 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
// Set new configs
setupQueueConfigs(clusterResource);
- initializeLeafQueueConfigs();
-
// run reinitialize on each existing queue, to trigger absolute cap
// recomputations
for (CSQueue res : this.getChildQueues()) {
@@ -87,72 +67,29 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
* Initialize leaf queue configs from template configurations specified on
* parent queue.
*/
- protected void initializeLeafQueueConfigs() {
+ protected AutoCreatedLeafQueueTemplate.Builder initializeLeafQueueConfigs
+ (String queuePath) {
CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- final String queuePath = super.getQueuePath();
+ AutoCreatedLeafQueueTemplate.Builder leafQueueTemplateBuilder = new
+ AutoCreatedLeafQueueTemplate.Builder();
int maxApps = conf.getMaximumApplicationsPerQueue(queuePath);
if (maxApps < 0) {
maxApps = (int) (
CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_SYSTEM_APPLICATIIONS
* getAbsoluteCapacity());
}
- userLimit = conf.getUserLimit(queuePath);
- userLimitFactor = conf.getUserLimitFactor(queuePath);
- maxAppsForAutoCreatedQueues = maxApps;
- maxAppsPerUserForAutoCreatedQueues =
- (int) (maxApps * (userLimit / 100.0f) * userLimitFactor);
-
- }
-
- /**
- * Number of maximum applications for each of the auto created leaf queues.
- *
- * @return maxAppsForAutoCreatedQueues
- */
- public int getMaxApplicationsForAutoCreatedQueues() {
- return maxAppsForAutoCreatedQueues;
- }
-
- /**
- * Number of maximum applications per user for each of the auto created
- * leaf queues.
- *
- * @return maxAppsPerUserForAutoCreatedQueues
- */
- public int getMaxApplicationsPerUserForAutoCreatedQueues() {
- return maxAppsPerUserForAutoCreatedQueues;
- }
-
- /**
- * User limit value for each of the auto created leaf queues.
- *
- * @return userLimit
- */
- public int getUserLimitForAutoCreatedQueues() {
- return userLimit;
- }
-
- /**
- * User limit factor value for each of the auto created leaf queues.
- *
- * @return userLimitFactor
- */
- public float getUserLimitFactor() {
- return userLimitFactor;
- }
- public int getMaxAppsForAutoCreatedQueues() {
- return maxAppsForAutoCreatedQueues;
- }
+ int userLimit = conf.getUserLimit(queuePath);
+ float userLimitFactor = conf.getUserLimitFactor(queuePath);
+ leafQueueTemplateBuilder.userLimit(userLimit)
+ .userLimitFactor(userLimitFactor)
+ .maxApps(maxApps)
+ .maxAppsPerUser(
+ (int) (maxApps * (userLimit / 100.0f) * userLimitFactor));
- public int getMaxAppsPerUserForAutoCreatedQueues() {
- return maxAppsPerUserForAutoCreatedQueues;
- }
-
- public int getUserLimit() {
- return userLimit;
+ return leafQueueTemplateBuilder;
}
/**
@@ -229,4 +166,111 @@ public abstract class AbstractManagedParentQueue extends ParentQueue {
}
return childQueue;
}
+
+ protected float sumOfChildCapacities() {
+ try {
+ writeLock.lock();
+ float ret = 0;
+ for (CSQueue l : childQueues) {
+ ret += l.getCapacity();
+ }
+ return ret;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ protected float sumOfChildAbsCapacities() {
+ try {
+ writeLock.lock();
+ float ret = 0;
+ for (CSQueue l : childQueues) {
+ ret += l.getAbsoluteCapacity();
+ }
+ return ret;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public static class AutoCreatedLeafQueueTemplate {
+
+ private QueueCapacities queueCapacities;
+
+ private int maxApps;
+ private int maxAppsPerUser;
+ private int userLimit;
+ private float userLimitFactor;
+
+ AutoCreatedLeafQueueTemplate(Builder builder) {
+ this.maxApps = builder.maxApps;
+ this.maxAppsPerUser = builder.maxAppsPerUser;
+ this.userLimit = builder.userLimit;
+ this.userLimitFactor = builder.userLimitFactor;
+ this.queueCapacities = builder.queueCapacities;
+ }
+
+ public static class Builder {
+ private int maxApps;
+ private int maxAppsPerUser;
+
+ private int userLimit;
+ private float userLimitFactor;
+
+ private QueueCapacities queueCapacities;
+
+ Builder maxApps(int maxApplications) {
+ this.maxApps = maxApplications;
+ return this;
+ }
+
+ Builder maxAppsPerUser(int maxApplicationsPerUser) {
+ this.maxAppsPerUser = maxApplicationsPerUser;
+ return this;
+ }
+
+ Builder userLimit(int usrLimit) {
+ this.userLimit = usrLimit;
+ return this;
+ }
+
+ Builder userLimitFactor(float ulf) {
+ this.userLimitFactor = ulf;
+ return this;
+ }
+
+ Builder capacities(QueueCapacities capacities) {
+ this.queueCapacities = capacities;
+ return this;
+ }
+
+ AutoCreatedLeafQueueTemplate build() {
+ return new AutoCreatedLeafQueueTemplate(this);
+ }
+ }
+
+ public int getUserLimit() {
+ return userLimit;
+ }
+
+ public float getUserLimitFactor() {
+ return userLimitFactor;
+ }
+
+ public QueueCapacities getQueueCapacities() {
+ return queueCapacities;
+ }
+
+ public int getMaxApps() {
+ return maxApps;
+ }
+
+ public int getMaxAppsPerUser() {
+ return maxAppsPerUser;
+ }
+ }
+
+ public AutoCreatedLeafQueueTemplate getLeafQueueTemplate() {
+ return leafQueueTemplate;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
index 4eb7cdd..bc206d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueue.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+ .AbstractManagedParentQueue.AutoCreatedLeafQueueTemplate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,17 +44,18 @@ public class AutoCreatedLeafQueue extends LeafQueue {
AbstractManagedParentQueue parent) throws IOException {
super(cs, queueName, parent, null);
- updateApplicationAndUserLimits(parent.getUserLimitForAutoCreatedQueues(),
- parent.getUserLimitFactor(),
- parent.getMaxApplicationsForAutoCreatedQueues(),
- parent.getMaxApplicationsPerUserForAutoCreatedQueues());
-
+ AutoCreatedLeafQueueTemplate leafQueueTemplate =
+ parent.getLeafQueueTemplate();
+ updateApplicationAndUserLimits(leafQueueTemplate.getUserLimit(),
+ leafQueueTemplate.getUserLimitFactor(),
+ leafQueueTemplate.getMaxApps(),
+ leafQueueTemplate.getMaxAppsPerUser());
this.parent = parent;
}
@Override
- public void reinitialize(CSQueue newlyParsedQueue,
- Resource clusterResource) throws IOException {
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+ throws IOException {
try {
writeLock.lock();
@@ -62,10 +65,12 @@ public class AutoCreatedLeafQueue extends LeafQueue {
CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
this, labelManager, null);
- updateApplicationAndUserLimits(parent.getUserLimitForAutoCreatedQueues(),
- parent.getUserLimitFactor(),
- parent.getMaxApplicationsForAutoCreatedQueues(),
- parent.getMaxApplicationsPerUserForAutoCreatedQueues());
+ AutoCreatedLeafQueueTemplate leafQueueTemplate =
+ parent.getLeafQueueTemplate();
+ updateApplicationAndUserLimits(leafQueueTemplate.getUserLimit(),
+ leafQueueTemplate.getUserLimitFactor(),
+ leafQueueTemplate.getMaxApps(),
+ leafQueueTemplate.getMaxAppsPerUser());
} finally {
writeLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 6f630f8..ed30ad1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -41,7 +41,6 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AccessControlException;
-import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -64,10 +63,10 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.placement.ApplicationPlacementContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementFactory;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationConstants;
@@ -146,6 +145,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.SettableFuture;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QUEUE_MAPPING;
+
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -560,44 +561,17 @@ public class CapacityScheduler extends
}
@VisibleForTesting
- public UserGroupMappingPlacementRule
- getUserGroupMappingPlacementRule() throws IOException {
+ public PlacementRule getUserGroupMappingPlacementRule() throws IOException {
try {
readLock.lock();
- boolean overrideWithQueueMappings = conf.getOverrideWithQueueMappings();
- LOG.info(
- "Initialized queue mappings, override: " + overrideWithQueueMappings);
-
- // Get new user/group mappings
- List<QueueMapping> newMappings = conf.getQueueMappings();
- // check if mappings refer to valid queues
- for (QueueMapping mapping : newMappings) {
- String mappingQueue = mapping.getQueue();
- if (!mappingQueue.equals(
- UserGroupMappingPlacementRule.CURRENT_USER_MAPPING) && !mappingQueue
- .equals(UserGroupMappingPlacementRule.PRIMARY_GROUP_MAPPING)) {
- CSQueue queue = getQueue(mappingQueue);
- if (queue == null || !(queue instanceof LeafQueue)) {
- throw new IOException(
- "mapping contains invalid or non-leaf queue " + mappingQueue);
- }
- }
- }
-
- // initialize groups if mappings are present
- if (newMappings.size() > 0) {
- Groups groups = new Groups(conf);
- return new UserGroupMappingPlacementRule(overrideWithQueueMappings,
- newMappings, groups);
- }
-
- return null;
+ return UserGroupMappingPlacementRule.get(this);
} finally {
readLock.unlock();
}
}
- private void updatePlacementRules() throws IOException {
+ @VisibleForTesting
+ void updatePlacementRules() throws IOException {
// Initialize placement rules
Collection<String> placementRuleStrs = conf.getStringCollection(
YarnConfiguration.QUEUE_PLACEMENT_RULES);
@@ -731,37 +705,92 @@ public class CapacityScheduler extends
}
}
- private void addApplication(ApplicationId applicationId,
- String queueName, String user, Priority priority) {
+ private void addApplication(ApplicationId applicationId, String queueName,
+ String user, Priority priority,
+ ApplicationPlacementContext placementContext) {
try {
writeLock.lock();
if (isSystemAppsLimitReached()) {
String message = "Maximum system application limit reached,"
+ "cannot accept submission of application: " + applicationId;
- this.rmContext.getDispatcher().getEventHandler().handle(new RMAppEvent(
- applicationId, RMAppEventType.APP_REJECTED, message));
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
+ message));
return;
}
// Sanity checks.
CSQueue queue = getQueue(queueName);
+
+ if (queue == null && placementContext != null) {
+ //Could be a potential auto-created leaf queue
+ try {
+ queue = autoCreateLeafQueue(placementContext);
+ } catch (YarnException | IOException e) {
+ LOG.error("Could not auto-create leaf queue due to : ", e);
+ final String message =
+ "Application " + applicationId + " submission by user : " + user
+ + " to queue : " + queueName + " failed : " + e.getMessage();
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
+ message));
+ }
+ }
+
if (queue == null) {
- String message =
+ final String message =
"Application " + applicationId + " submitted by user " + user
+ " to unknown queue: " + queueName;
+
this.rmContext.getDispatcher().getEventHandler().handle(
new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
message));
return;
}
+
if (!(queue instanceof LeafQueue)) {
String message =
- "Application " + applicationId + " submitted by user " + user
- + " to non-leaf queue: " + queueName;
+ "Application " + applicationId + " submitted by user : " + user
+ + " to non-leaf queue : " + queueName;
this.rmContext.getDispatcher().getEventHandler().handle(
new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
message));
return;
+ } else if (queue instanceof AutoCreatedLeafQueue && queue
+ .getParent() instanceof ManagedParentQueue) {
+
+ //If queue already exists and auto-queue creation was not required,
+ //placement context should not be null
+ if (placementContext == null) {
+ String message =
+ "Application " + applicationId + " submission by user : " + user
+ + " to specified queue : " + queueName + " is prohibited. "
+ + "Verify automatic queue mapping for user exists in " +
+ QUEUE_MAPPING;
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
+ message));
+ return;
+ // For a queue which exists already and
+ // not auto-created above, then its parent queue should match
+ // the parent queue specified in queue mapping
+ } else if (!queue.getParent().getQueueName().equals(
+ placementContext.getParentQueue())) {
+ String message =
+ "Auto created Leaf queue " + placementContext.getQueue() + " "
+ + "already exists under queue : " + queue
+ .getParent().getQueuePath()
+ + ".But Queue mapping configuration " +
+ CapacitySchedulerConfiguration.QUEUE_MAPPING + " has been "
+ + "updated to a different parent queue : "
+ + placementContext.getParentQueue()
+ + " for the specified user : " + user;
+ this.rmContext.getDispatcher().getEventHandler().handle(
+ new RMAppEvent(applicationId, RMAppEventType.APP_REJECTED,
+ message));
+ return;
+ }
}
+
// Submit to the queue
try {
queue.submitApplication(applicationId, user, queueName);
@@ -1483,7 +1512,8 @@ public class CapacityScheduler extends
if (queueName != null) {
if (!appAddedEvent.getIsAppRecovering()) {
addApplication(appAddedEvent.getApplicationId(), queueName,
- appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority());
+ appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority(),
+ appAddedEvent.getPlacementContext());
} else {
addApplicationOnRecovery(appAddedEvent.getApplicationId(), queueName,
appAddedEvent.getUser(), appAddedEvent.getApplicatonPriority());
@@ -2001,7 +2031,8 @@ public class CapacityScheduler extends
try {
writeLock.lock();
LeafQueue queue = this.queueManager.getAndCheckLeafQueue(inQueue);
- ParentQueue parent = (ParentQueue) queue.getParent();
+ AbstractManagedParentQueue parent = (AbstractManagedParentQueue) queue
+ .getParent();
if (!(queue instanceof AutoCreatedLeafQueue)) {
throw new SchedulerDynamicEditException(
@@ -2010,7 +2041,8 @@ public class CapacityScheduler extends
}
if (parent == null
- || !(AbstractManagedParentQueue.class.isAssignableFrom(parent.getClass()))) {
+ || !(AbstractManagedParentQueue.class.isAssignableFrom(
+ parent.getClass()))) {
throw new SchedulerDynamicEditException(
"The parent of AutoCreatedLeafQueue " + inQueue
+ " must be a PlanQueue/ManagedParentQueue");
@@ -2655,4 +2687,43 @@ public class CapacityScheduler extends
}
return null;
}
+
+ private LeafQueue autoCreateLeafQueue(
+ ApplicationPlacementContext placementContext)
+ throws IOException, YarnException {
+
+ AutoCreatedLeafQueue autoCreatedLeafQueue = null;
+
+ String leafQueueName = placementContext.getQueue();
+ String parentQueueName = placementContext.getParentQueue();
+
+ if (!StringUtils.isEmpty(parentQueueName)) {
+ CSQueue parentQueue = getQueue(parentQueueName);
+
+ if (parentQueue != null && conf.isAutoCreateChildQueueEnabled(
+ parentQueue.getQueuePath())) {
+
+ ManagedParentQueue autoCreateEnabledParentQueue =
+ (ManagedParentQueue) parentQueue;
+ autoCreatedLeafQueue = new AutoCreatedLeafQueue(this, leafQueueName,
+ autoCreateEnabledParentQueue);
+
+ addQueue(autoCreatedLeafQueue);
+
+ //TODO - Set entitlement through capacity management policy
+ } else{
+ throw new SchedulerDynamicEditException(
+ "Could not auto-create leaf queue for " + leafQueueName
+ + ". Queue mapping specifies an invalid parent queue "
+ + "which does not exist "
+ + parentQueueName);
+ }
+ } else{
+ throw new SchedulerDynamicEditException(
+ "Could not auto-create leaf queue for " + leafQueueName
+ + ". Queue mapping does not specify"
+ + " which parent queue it needs to be created under.");
+ }
+ return autoCreatedLeafQueue;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index bfead35..4515453 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -907,6 +907,12 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
DEFAULT_ENABLE_QUEUE_MAPPING_OVERRIDE);
}
+ @Private
+ @VisibleForTesting
+ public void setOverrideWithQueueMappings(boolean overrideWithQueueMappings) {
+ setBoolean(ENABLE_QUEUE_MAPPING_OVERRIDE, overrideWithQueueMappings);
+ }
+
/**
* Returns a collection of strings, trimming leading and trailing whitespeace
* on each value
@@ -981,6 +987,31 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
return mappings;
}
+ @Private
+ @VisibleForTesting
+ public void setQueuePlacementRules(Collection<String> queuePlacementRules) {
+ if (queuePlacementRules == null) {
+ return;
+ }
+ String str = StringUtils.join(",", queuePlacementRules);
+ setStrings(YarnConfiguration.QUEUE_PLACEMENT_RULES, str);
+ }
+
+ @Private
+ @VisibleForTesting
+ public void setQueueMappings(List<QueueMapping> queueMappings) {
+ if (queueMappings == null) {
+ return;
+ }
+
+ List<String> queueMappingStrs = new ArrayList<>();
+ for (QueueMapping mapping : queueMappings) {
+ queueMappingStrs.add(mapping.toString());
+ }
+
+ setStrings(QUEUE_MAPPING, StringUtils.join(",", queueMappingStrs));
+ }
+
public boolean isReservable(String queue) {
boolean isReservable =
getBoolean(getQueuePrefix(queue) + IS_RESERVABLE, false);
@@ -1523,4 +1554,126 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
public void setDefaultLifetimePerQueue(String queue, long defaultLifetime) {
setLong(getQueuePrefix(queue) + DEFAULT_LIFETIME_SUFFIX, defaultLifetime);
}
+
+ @Private
+ public static final boolean DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED = false;
+
+ @Private
+ public static final String AUTO_CREATE_CHILD_QUEUE_ENABLED =
+ "auto-create-child-queue.enabled";
+
+ @Private
+ public static final String AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX =
+ "leaf-queue-template";
+
+ @Private
+ public static final String AUTO_CREATE_QUEUE_MAX_QUEUES =
+ "auto-create-child-queue.max-queues";
+
+ @Private
+ public static final int DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES = 1000;
+
+ /**
+ * If true, this queue will be created as a Parent Queue which Auto Created
+ * leaf child queues
+ *
+ * @param queuePath The queues path
+ * @return true if auto create is enabled for child queues else false. Default
+ * is false
+ */
+ @Private
+ public boolean isAutoCreateChildQueueEnabled(String queuePath) {
+ boolean isAutoCreateEnabled = getBoolean(
+ getQueuePrefix(queuePath) + AUTO_CREATE_CHILD_QUEUE_ENABLED,
+ DEFAULT_AUTO_CREATE_CHILD_QUEUE_ENABLED);
+ return isAutoCreateEnabled;
+ }
+
+ @Private
+ @VisibleForTesting
+ public void setAutoCreateChildQueueEnabled(String queuePath,
+ boolean autoCreationEnabled) {
+ setBoolean(getQueuePrefix(queuePath) +
+ AUTO_CREATE_CHILD_QUEUE_ENABLED,
+ autoCreationEnabled);
+ }
+
+ /**
+ * Get the auto created leaf queue's template configuration prefix
+ * Leaf queue's template capacities are configured at the parent queue
+ *
+ * @param queuePath parent queue's path
+ * @return Config prefix for leaf queue template configurations
+ */
+ @Private
+ public String getAutoCreatedQueueTemplateConfPrefix(String queuePath) {
+ return queuePath + DOT + AUTO_CREATED_LEAF_QUEUE_TEMPLATE_PREFIX;
+ }
+
+ @Private
+ public static final String FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY =
+ "auto-create-child-queue.fail-on-exceeding-parent-capacity";
+
+ @Private
+ public static final boolean DEFAULT_FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY =
+ false;
+
+ /**
+ * Fail further auto leaf queue creation when parent's guaranteed capacity is
+ * exceeded.
+ *
+ * @param queuePath the parent queue's path
+ * @return true if configured to fail else false
+ */
+ @Private
+ public boolean getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
+ String queuePath) {
+ boolean shouldFailAutoQueueCreationOnExceedingGuaranteedCapacity =
+ getBoolean(getQueuePrefix(queuePath)
+ + FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY,
+ DEFAULT_FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY);
+ return shouldFailAutoQueueCreationOnExceedingGuaranteedCapacity;
+ }
+
+ @VisibleForTesting
+ @Private
+ public void setShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
+ String queuePath, boolean autoCreationEnabled) {
+ setBoolean(
+ getQueuePrefix(queuePath) +
+ FAIL_AUTO_CREATION_ON_EXCEEDING_CAPACITY,
+ autoCreationEnabled);
+ }
+
+ /**
+ * Get the max number of leaf queues that are allowed to be created under
+ * a parent queue
+ *
+ * @param queuePath the paret queue's path
+ * @return the max number of leaf queues allowed to be auto created
+ */
+ @Private
+ public int getAutoCreatedQueuesMaxChildQueuesLimit(String queuePath) {
+ return getInt(getQueuePrefix(queuePath) +
+ AUTO_CREATE_QUEUE_MAX_QUEUES,
+ DEFAULT_AUTO_CREATE_QUEUE_MAX_QUEUES);
+ }
+
+ @Private
+ @VisibleForTesting
+ public void setAutoCreatedLeafQueueTemplateCapacity(String queuePath,
+ float val) {
+ String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ queuePath);
+ setCapacity(leafQueueConfPrefix, val);
+ }
+
+ @Private
+ @VisibleForTesting
+ public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath,
+ float val) {
+ String leafQueueConfPrefix = getAutoCreatedQueueTemplateConfPrefix(
+ queuePath);
+ setMaximumCapacity(leafQueueConfPrefix, val);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
index 7be2529..eb50123 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerQueueManager.java
@@ -154,7 +154,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
* @throws IOException if fails to initialize queues
*/
public void initializeQueues(CapacitySchedulerConfiguration conf)
- throws IOException {
+ throws IOException {
root = parseQueue(this.csContext, conf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues, NOOP);
setQueueAcls(authorizer, appPriorityACLManager, queues);
@@ -176,7 +176,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
if (!csContext.isConfigurationMutable() ||
csContext.getRMContext().getHAServiceState()
!= HAServiceProtocol.HAServiceState.STANDBY) {
- // Ensure queue hiearchy in the new XML file is proper.
+ // Ensure queue hierarchy in the new XML file is proper.
validateQueueHierarchy(queues, newQueues);
}
@@ -216,11 +216,13 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
Map<String, CSQueue> oldQueues,
QueueHook hook) throws IOException {
CSQueue queue;
- String fullQueueName =
- (parent == null) ? queueName
- : (parent.getQueuePath() + "." + queueName);
+ String fullQueueName = (parent == null) ?
+ queueName :
+ (parent.getQueuePath() + "." + queueName);
String[] childQueueNames = conf.getQueues(fullQueueName);
boolean isReservableQueue = conf.isReservable(fullQueueName);
+ boolean isAutoCreateEnabled = conf.isAutoCreateChildQueueEnabled(
+ fullQueueName);
if (childQueueNames == null || childQueueNames.length == 0) {
if (null == parent) {
throw new IllegalStateException(
@@ -229,9 +231,8 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
// Check if the queue will be dynamically managed by the Reservation
// system
if (isReservableQueue) {
- queue =
- new PlanQueue(csContext, queueName, parent,
- oldQueues.get(queueName));
+ queue = new PlanQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
//initializing the "internal" default queue, for SLS compatibility
String defReservationId =
@@ -249,38 +250,46 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
((PlanQueue) queue).setChildQueues(childQueues);
queues.put(defReservationId, resQueue);
- } else {
- queue =
- new LeafQueue(csContext, queueName, parent,
- oldQueues.get(queueName));
+ } else if (isAutoCreateEnabled) {
+ queue = new ManagedParentQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
+ } else{
+ queue = new LeafQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
// Used only for unit tests
queue = hook.hook(queue);
}
- } else {
+ } else{
if (isReservableQueue) {
throw new IllegalStateException(
"Only Leaf Queues can be reservable for " + queueName);
}
- ParentQueue parentQueue =
- new ParentQueue(csContext, queueName, parent,
- oldQueues.get(queueName));
+
+ ParentQueue parentQueue;
+ if (isAutoCreateEnabled) {
+ parentQueue = new ManagedParentQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
+ } else{
+ parentQueue = new ParentQueue(csContext, queueName, parent,
+ oldQueues.get(queueName));
+ }
// Used only for unit tests
queue = hook.hook(parentQueue);
List<CSQueue> childQueues = new ArrayList<>();
for (String childQueueName : childQueueNames) {
- CSQueue childQueue =
- parseQueue(csContext, conf, queue, childQueueName,
- queues, oldQueues, hook);
+ CSQueue childQueue = parseQueue(csContext, conf, queue, childQueueName,
+ queues, oldQueues, hook);
childQueues.add(childQueue);
}
parentQueue.setChildQueues(childQueues);
+
}
- if (queue instanceof LeafQueue && queues.containsKey(queueName)
- && queues.get(queueName) instanceof LeafQueue) {
+ if (queue instanceof LeafQueue && queues.containsKey(queueName) && queues
+ .get(queueName) instanceof LeafQueue) {
throw new IOException("Two leaf queues were named " + queueName
+ ". Leaf queue names must be distinct");
}
@@ -312,27 +321,46 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
if (oldQueue.getState() == QueueState.STOPPED) {
LOG.info("Deleting Queue " + queueName + ", as it is not"
+ " present in the modified capacity configuration xml");
- } else {
+ } else{
throw new IOException(oldQueue.getQueuePath() + " is deleted from"
+ " the new capacity scheduler configuration, but the"
- + " queue is not yet in stopped state. "
- + "Current State : " + oldQueue.getState());
+ + " queue is not yet in stopped state. " + "Current State : "
+ + oldQueue.getState());
}
} else if (!oldQueue.getQueuePath().equals(newQueue.getQueuePath())) {
//Queue's cannot be moved from one hierarchy to other
- throw new IOException(queueName + " is moved from:"
- + oldQueue.getQueuePath() + " to:" + newQueue.getQueuePath()
- + " after refresh, which is not allowed.");
- } else if (oldQueue instanceof LeafQueue
+ throw new IOException(
+ queueName + " is moved from:" + oldQueue.getQueuePath() + " to:"
+ + newQueue.getQueuePath()
+ + " after refresh, which is not allowed.");
+ } else if (oldQueue instanceof ParentQueue
+ && !(oldQueue instanceof ManagedParentQueue)
+ && newQueue instanceof ManagedParentQueue) {
+ throw new IOException(
+ "Can not convert parent queue: " + oldQueue.getQueuePath()
+ + " to auto create enabled parent queue since "
+ + "it could have other pre-configured queues which is not "
+ + "supported");
+ } else if (oldQueue instanceof ManagedParentQueue
+ && !(newQueue instanceof ManagedParentQueue)) {
+ throw new IOException(
+ "Cannot convert auto create enabled parent queue: " + oldQueue
+ .getQueuePath() + " to leaf queue. Please check "
+ + " parent queue's configuration "
+ + CapacitySchedulerConfiguration
+ .AUTO_CREATE_CHILD_QUEUE_ENABLED
+ + " is set to true");
+ } else if (oldQueue instanceof LeafQueue
&& newQueue instanceof ParentQueue) {
if (oldQueue.getState() == QueueState.STOPPED) {
LOG.info("Converting the leaf queue: " + oldQueue.getQueuePath()
+ " to parent queue.");
- } else {
- throw new IOException("Can not convert the leaf queue: "
- + oldQueue.getQueuePath() + " to parent queue since "
- + "it is not yet in stopped state. Current State : "
- + oldQueue.getState());
+ } else{
+ throw new IOException(
+ "Can not convert the leaf queue: " + oldQueue.getQueuePath()
+ + " to parent queue since "
+ + "it is not yet in stopped state. Current State : "
+ + oldQueue.getState());
}
} else if (oldQueue instanceof ParentQueue
&& newQueue instanceof LeafQueue) {
@@ -352,6 +380,7 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
*/
private void updateQueues(Map<String, CSQueue> existingQueues,
Map<String, CSQueue> newQueues) {
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
for (Map.Entry<String, CSQueue> e : newQueues.entrySet()) {
String queueName = e.getKey();
CSQueue queue = e.getValue();
@@ -363,7 +392,13 @@ public class CapacitySchedulerQueueManager implements SchedulerQueueManager<
.iterator(); itr.hasNext();) {
Map.Entry<String, CSQueue> e = itr.next();
String queueName = e.getKey();
- if (!newQueues.containsKey(queueName)) {
+ CSQueue existingQueue = e.getValue();
+
+ //TODO - Handle case when auto create is disabled on parent queues
+ if (!newQueues.containsKey(queueName) && !(
+ existingQueue instanceof AutoCreatedLeafQueue && conf
+ .isAutoCreateChildQueueEnabled(
+ existingQueue.getParent().getQueuePath()))) {
itr.remove();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
new file mode 100644
index 0000000..ff795e4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ManagedParentQueue.java
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler
+ .SchedulerDynamicEditException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * Auto Creation enabled Parent queue. This queue initially does not have any
+ * children to start with and all child
+ * leaf queues will be auto created. Currently this does not allow other
+ * pre-configured leaf or parent queues to
+ * co-exist along with auto-created leaf queues. The auto creation is limited
+ * to leaf queues currently.
+ */
+public class ManagedParentQueue extends AbstractManagedParentQueue {
+
+ private boolean shouldFailAutoCreationWhenGuaranteedCapacityExceeded = false;
+
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ManagedParentQueue.class);
+
+ public ManagedParentQueue(final CapacitySchedulerContext cs,
+ final String queueName, final CSQueue parent, final CSQueue old)
+ throws IOException {
+ super(cs, queueName, parent, old);
+ String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
+ csContext.getConfiguration());
+ this.leafQueueTemplate = initializeLeafQueueConfigs(
+ leafQueueTemplateConfPrefix).build();
+
+ StringBuffer queueInfo = new StringBuffer();
+ queueInfo.append("Created Managed Parent Queue: ").append(queueName).append(
+ "]\nwith capacity: [").append(super.getCapacity()).append(
+ "]\nwith max capacity: [").append(super.getMaximumCapacity()).append(
+ "\nwith max apps: [").append(leafQueueTemplate.getMaxApps()).append(
+ "]\nwith max apps per user: [").append(
+ leafQueueTemplate.getMaxAppsPerUser()).append("]\nwith user limit: [")
+ .append(leafQueueTemplate.getUserLimit()).append(
+ "]\nwith user limit factor: [").append(
+ leafQueueTemplate.getUserLimitFactor()).append("].");
+ LOG.info(queueInfo.toString());
+ }
+
+ @Override
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
+ throws IOException {
+ validate(newlyParsedQueue);
+ super.reinitialize(newlyParsedQueue, clusterResource);
+ String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(
+ csContext.getConfiguration());
+ this.leafQueueTemplate = initializeLeafQueueConfigs(
+ leafQueueTemplateConfPrefix).build();
+ }
+
+ @Override
+ protected AutoCreatedLeafQueueTemplate.Builder initializeLeafQueueConfigs(
+ String queuePath) {
+
+ AutoCreatedLeafQueueTemplate.Builder leafQueueTemplate =
+ super.initializeLeafQueueConfigs(queuePath);
+
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ String leafQueueTemplateConfPrefix = getLeafQueueConfigPrefix(conf);
+ QueueCapacities queueCapacities = new QueueCapacities(false);
+ CSQueueUtils.loadUpdateAndCheckCapacities(leafQueueTemplateConfPrefix,
+ csContext.getConfiguration(), queueCapacities, getQueueCapacities());
+ leafQueueTemplate.capacities(queueCapacities);
+
+ shouldFailAutoCreationWhenGuaranteedCapacityExceeded =
+ conf.getShouldFailAutoQueueCreationWhenGuaranteedCapacityExceeded(
+ getQueuePath());
+
+ return leafQueueTemplate;
+ }
+
+ protected void validate(final CSQueue newlyParsedQueue) throws IOException {
+ // Sanity check
+ if (!(newlyParsedQueue instanceof ManagedParentQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
+ }
+ }
+
+ @Override
+ public void addChildQueue(CSQueue childQueue)
+ throws SchedulerDynamicEditException {
+ try {
+ writeLock.lock();
+
+ if (childQueue == null || !(childQueue instanceof AutoCreatedLeafQueue)) {
+ throw new SchedulerDynamicEditException(
+ "Expected child queue to be an instance of AutoCreatedLeafQueue");
+ }
+
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ ManagedParentQueue parentQueue =
+ (ManagedParentQueue) childQueue.getParent();
+
+ String leafQueueName = childQueue.getQueueName();
+ int maxQueues = conf.getAutoCreatedQueuesMaxChildQueuesLimit(
+ parentQueue.getQueuePath());
+
+ if (parentQueue.getChildQueues().size() >= maxQueues) {
+ throw new SchedulerDynamicEditException(
+ "Cannot auto create leaf queue " + leafQueueName + ".Max Child "
+ + "Queue limit exceeded which is configured as : " + maxQueues
+ + " and number of child queues is : " + parentQueue
+ .getChildQueues().size());
+ }
+
+ if (shouldFailAutoCreationWhenGuaranteedCapacityExceeded) {
+ if (getLeafQueueTemplate().getQueueCapacities().getAbsoluteCapacity()
+ + parentQueue.sumOfChildAbsCapacities() > parentQueue
+ .getAbsoluteCapacity()) {
+ throw new SchedulerDynamicEditException(
+ "Cannot auto create leaf queue " + leafQueueName + ". Child "
+ + "queues capacities have reached parent queue : "
+ + parentQueue.getQueuePath() + " guaranteed capacity");
+ }
+ }
+
+ AutoCreatedLeafQueue leafQueue = (AutoCreatedLeafQueue) childQueue;
+ super.addChildQueue(leafQueue);
+ //TODO - refresh policy queue after capacity management is added
+
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private String getLeafQueueConfigPrefix(CapacitySchedulerConfiguration conf) {
+ return conf.getAutoCreatedQueueTemplateConfPrefix(getQueuePath());
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index d61951b..959ca51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -1081,17 +1081,4 @@ public class ParentQueue extends AbstractCSQueue {
public QueueOrderingPolicy getQueueOrderingPolicy() {
return queueOrderingPolicy;
}
-
- protected float sumOfChildCapacities() {
- try {
- writeLock.lock();
- float ret = 0;
- for (CSQueue l : childQueues) {
- ret += l.getCapacity();
- }
- return ret;
- } finally {
- writeLock.unlock();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0987a7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 4ab2e9f..b7f8aa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -40,6 +40,19 @@ public class PlanQueue extends AbstractManagedParentQueue {
public PlanQueue(CapacitySchedulerContext cs, String queueName,
CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
+ this.leafQueueTemplate = initializeLeafQueueConfigs(getQueuePath()).build();
+
+ StringBuffer queueInfo = new StringBuffer();
+ queueInfo.append("Created Plan Queue: ").append(queueName).append(
+ "]\nwith capacity: [").append(super.getCapacity()).append(
+ "]\nwith max capacity: [").append(super.getMaximumCapacity()).append(
+ "\nwith max apps: [").append(leafQueueTemplate.getMaxApps()).append(
+ "]\nwith max apps per user: [").append(
+ leafQueueTemplate.getMaxAppsPerUser()).append("]\nwith user limit: [")
+ .append(leafQueueTemplate.getUserLimit()).append(
+ "]\nwith user limit factor: [").append(
+ leafQueueTemplate.getUserLimitFactor()).append("].");
+ LOG.info(queueInfo.toString());
}
@Override
@@ -47,17 +60,21 @@ public class PlanQueue extends AbstractManagedParentQueue {
throws IOException {
validate(newlyParsedQueue);
super.reinitialize(newlyParsedQueue, clusterResource);
+ this.leafQueueTemplate = initializeLeafQueueConfigs(getQueuePath()).build();
}
@Override
- protected void initializeLeafQueueConfigs() {
- String queuePath = super.getQueuePath();
+ protected AutoCreatedLeafQueueTemplate.Builder initializeLeafQueueConfigs
+ (String queuePath) {
+ AutoCreatedLeafQueueTemplate.Builder leafQueueTemplate = super
+ .initializeLeafQueueConfigs
+ (queuePath);
showReservationsAsQueues = csContext.getConfiguration()
.getShowReservationAsQueues(queuePath);
- super.initializeLeafQueueConfigs();
+ return leafQueueTemplate;
}
- private void validate(final CSQueue newlyParsedQueue) throws IOException {
+ protected void validate(final CSQueue newlyParsedQueue) throws IOException {
// Sanity check
if (!(newlyParsedQueue instanceof PlanQueue) || !newlyParsedQueue
.getQueuePath().equals(getQueuePath())) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/19] hadoop git commit: HDFS-12801. RBF: Set MountTableResolver as
default file resolver. Contributed by Inigo Goiri.
Posted by as...@apache.org.
HDFS-12801. RBF: Set MountTableResolver as default file resolver. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e182e777
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e182e777
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e182e777
Branch: refs/heads/YARN-6592
Commit: e182e777947a85943504a207deb3cf3ffc047910
Parents: 0987a7b
Author: Inigo Goiri <in...@apache.org>
Authored: Thu Nov 16 16:58:47 2017 -0800
Committer: Inigo Goiri <in...@apache.org>
Committed: Thu Nov 16 16:58:47 2017 -0800
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e182e777/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 28621ba..7ff91f2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -4883,7 +4883,7 @@
<property>
<name>dfs.federation.router.file.resolver.client.class</name>
- <value>org.apache.hadoop.hdfs.server.federation.MockResolver</value>
+ <value>org.apache.hadoop.hdfs.server.federation.resolver.MountTableResolver</value>
<description>
Class to resolve files to subclusters.
</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/19] hadoop git commit: YARN-7390. All reservation related test
cases failed when TestYarnClient runs against Fair Scheduler. (Yufei Gu via
Haibo Chen)
Posted by as...@apache.org.
YARN-7390. All reservation related test cases failed when TestYarnClient runs against Fair Scheduler. (Yufei Gu via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28d0fcbe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28d0fcbe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28d0fcbe
Branch: refs/heads/YARN-6592
Commit: 28d0fcbef40930ca5652c0e9a5d777910f3ad3c4
Parents: 61ace17
Author: Haibo Chen <ha...@apache.org>
Authored: Thu Nov 16 10:48:24 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Thu Nov 16 10:48:24 2017 -0800
----------------------------------------------------------------------
.../yarn/client/api/impl/TestYarnClient.java | 455 ++--------------
.../api/impl/TestYarnClientWithReservation.java | 521 +++++++++++++++++++
2 files changed, 551 insertions(+), 425 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d0fcbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 4c1a9cf..f6e305f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -33,7 +33,6 @@ import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -42,7 +41,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import com.google.common.base.Supplier;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
@@ -75,14 +73,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -96,13 +86,7 @@ import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.ReservationDefinition;
-import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.ReservationRequest;
-import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
-import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@@ -119,23 +103,28 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
-import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.hadoop.yarn.util.UTCClock;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.slf4j.event.Level;
-public class TestYarnClient {
+/**
+ * This class is to test class {@link YarnClient) and {@link YarnClientImpl}.
+ */
+public class TestYarnClient extends ParameterizedSchedulerTestBase {
+
+ public TestYarnClient(SchedulerType type) throws IOException {
+ super(type);
+ }
+
+ protected void configureFairScheduler(YarnConfiguration conf) {}
@Before
public void setup() {
@@ -145,7 +134,7 @@ public class TestYarnClient {
@Test
public void testClientStop() {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
ResourceManager rm = new ResourceManager();
rm.init(conf);
rm.start();
@@ -159,7 +148,7 @@ public class TestYarnClient {
@Test
public void testStartWithTimelineV15() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.5f);
YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
@@ -254,7 +243,7 @@ public class TestYarnClient {
@SuppressWarnings("deprecation")
@Test (timeout = 30000)
public void testSubmitApplication() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
100); // speed up tests
final YarnClient client = new MockYarnClient();
@@ -301,7 +290,7 @@ public class TestYarnClient {
@SuppressWarnings("deprecation")
@Test (timeout = 20000)
public void testSubmitApplicationInterrupted() throws IOException {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
int pollIntervalMs = 1000;
conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
pollIntervalMs);
@@ -406,10 +395,9 @@ public class TestYarnClient {
rm.start();
RMApp app = rm.submitApp(2000);
- Configuration conf = new Configuration();
@SuppressWarnings("resource")
final YarnClient client = new MockYarnClient();
- client.init(conf);
+ client.init(getConf());
client.start();
client.killApplication(app.getApplicationId());
@@ -447,9 +435,8 @@ public class TestYarnClient {
@Test (timeout = 10000)
public void testGetApplications() throws YarnException, IOException {
- Configuration conf = new Configuration();
final YarnClient client = new MockYarnClient();
- client.init(conf);
+ client.init(getConf());
client.start();
List<ApplicationReport> expectedReports = ((MockYarnClient)client).getReports();
@@ -500,9 +487,8 @@ public class TestYarnClient {
@Test(timeout = 10000)
public void testGetApplicationAttempts() throws YarnException, IOException {
- Configuration conf = new Configuration();
final YarnClient client = new MockYarnClient();
- client.init(conf);
+ client.init(getConf());
client.start();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
@@ -539,7 +525,7 @@ public class TestYarnClient {
@Test(timeout = 10000)
public void testGetContainers() throws YarnException, IOException {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
true);
@@ -572,7 +558,7 @@ public class TestYarnClient {
@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
true);
final YarnClient client = new MockYarnClient();
@@ -603,9 +589,8 @@ public class TestYarnClient {
@Test (timeout = 10000)
public void testGetLabelsToNodes() throws YarnException, IOException {
- Configuration conf = new Configuration();
final YarnClient client = new MockYarnClient();
- client.init(conf);
+ client.init(getConf());
client.start();
// Get labels to nodes mapping
@@ -629,9 +614,8 @@ public class TestYarnClient {
@Test (timeout = 10000)
public void testGetNodesToLabels() throws YarnException, IOException {
- Configuration conf = new Configuration();
final YarnClient client = new MockYarnClient();
- client.init(conf);
+ client.init(getConf());
client.start();
// Get labels to nodes mapping
@@ -1025,7 +1009,7 @@ public class TestYarnClient {
MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
YarnClient rmClient = null;
try {
- cluster.init(new YarnConfiguration());
+ cluster.init(getConf());
cluster.start();
final Configuration yarnConf = cluster.getConfig();
rmClient = YarnClient.createYarnClient();
@@ -1146,7 +1130,7 @@ public class TestYarnClient {
boolean expectedTimeoutEnforcement) {
YarnClientImpl client = new YarnClientImpl();
try {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
if (valueForTimeout != null) {
conf.setLong(
YarnConfiguration.YARN_CLIENT_APPLICATION_CLIENT_PROTOCOL_POLL_TIMEOUT_MS,
@@ -1165,7 +1149,7 @@ public class TestYarnClient {
@Test
public void testBestEffortTimelineDelegationToken()
throws Exception {
- Configuration conf = new YarnConfiguration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
@@ -1199,7 +1183,7 @@ public class TestYarnClient {
@Test
public void testAutomaticTimelineDelegationTokenLoading()
throws Exception {
- Configuration conf = new YarnConfiguration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
TimelineDelegationTokenIdentifier timelineDT =
@@ -1289,7 +1273,7 @@ public class TestYarnClient {
public void testParseTimelineDelegationTokenRenewer() throws Exception {
// Client side
YarnClientImpl client = (YarnClientImpl) YarnClient.createYarnClient();
- Configuration conf = new YarnConfiguration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.set(YarnConfiguration.RM_PRINCIPAL, "rm/_HOST@EXAMPLE.COM");
conf.set(
@@ -1303,387 +1287,9 @@ public class TestYarnClient {
}
}
- private MiniYARNCluster setupMiniYARNCluster() throws Exception {
- CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- ReservationSystemTestUtil.setupQueueConfiguration(conf);
- conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
- conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
- MiniYARNCluster cluster =
- new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
-
- cluster.init(conf);
- cluster.start();
-
- GenericTestUtils.waitFor(new Supplier<Boolean>() {
- @Override
- public Boolean get() {
- return cluster.getResourceManager().getRMContext()
- .getReservationSystem()
- .getPlan(ReservationSystemTestUtil.reservationQ)
- .getTotalCapacity().getMemorySize() > 6000;
- }
- }, 10, 10000);
-
- return cluster;
- }
-
- private YarnClient setupYarnClient(MiniYARNCluster cluster) {
- final Configuration yarnConf = cluster.getConfig();
- YarnClient client = YarnClient.createYarnClient();
- client.init(yarnConf);
- client.start();
- return client;
- }
-
- private ReservationSubmissionRequest submitReservationTestHelper(
- YarnClient client, long arrival, long deadline, long duration)
- throws IOException, YarnException {
- ReservationId reservationID = client.createReservation().getReservationId();
- ReservationSubmissionRequest sRequest = createSimpleReservationRequest(
- reservationID, 4, arrival, deadline, duration);
- ReservationSubmissionResponse sResponse =
- client.submitReservation(sRequest);
- Assert.assertNotNull(sResponse);
- Assert.assertNotNull(reservationID);
- System.out.println("Submit reservation response: " + reservationID);
-
- return sRequest;
- }
-
- @Test
- public void testCreateReservation() throws Exception {
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- // Submit the reservation again with the same request and make sure it
- // passes.
- client.submitReservation(sRequest);
-
- // Submit the reservation with the same reservation id but different
- // reservation definition, and ensure YarnException is thrown.
- arrival = clock.getTime();
- ReservationDefinition rDef = sRequest.getReservationDefinition();
- rDef.setArrival(arrival + duration);
- sRequest.setReservationDefinition(rDef);
- try {
- client.submitReservation(sRequest);
- Assert.fail("Reservation submission should fail if a duplicate "
- + "reservation id is used, but the reservation definition has been "
- + "updated.");
- } catch (Exception e) {
- Assert.assertTrue(e instanceof YarnException);
- }
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- @Test
- public void testUpdateReservation() throws Exception {
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- ReservationDefinition rDef = sRequest.getReservationDefinition();
- ReservationRequest rr =
- rDef.getReservationRequests().getReservationResources().get(0);
- ReservationId reservationID = sRequest.getReservationId();
- rr.setNumContainers(5);
- arrival = clock.getTime();
- duration = 30000;
- deadline = (long) (arrival + 1.05 * duration);
- rr.setDuration(duration);
- rDef.setArrival(arrival);
- rDef.setDeadline(deadline);
- ReservationUpdateRequest uRequest =
- ReservationUpdateRequest.newInstance(rDef, reservationID);
- ReservationUpdateResponse uResponse = client.updateReservation(uRequest);
- Assert.assertNotNull(uResponse);
- System.out.println("Update reservation response: " + uResponse);
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- @Test
- public void testListReservationsByReservationId() throws Exception{
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- ReservationId reservationID = sRequest.getReservationId();
- ReservationListRequest request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, reservationID.toString(), -1,
- -1, false);
- ReservationListResponse response = client.listReservations(request);
- Assert.assertNotNull(response);
- Assert.assertEquals(1, response.getReservationAllocationState().size());
- Assert.assertEquals(response.getReservationAllocationState().get(0)
- .getReservationId().getId(), reservationID.getId());
- Assert.assertEquals(response.getReservationAllocationState().get(0)
- .getResourceAllocationRequests().size(), 0);
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- @Test
- public void testListReservationsByTimeInterval() throws Exception {
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- // List reservations, search by a point in time within the reservation
- // range.
- arrival = clock.getTime();
- ReservationId reservationID = sRequest.getReservationId();
- ReservationListRequest request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, "", arrival + duration / 2,
- arrival + duration / 2, true);
-
- ReservationListResponse response = client.listReservations(request);
- Assert.assertNotNull(response);
- Assert.assertEquals(1, response.getReservationAllocationState().size());
- Assert.assertEquals(response.getReservationAllocationState().get(0)
- .getReservationId().getId(), reservationID.getId());
- // List reservations, search by time within reservation interval.
- request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, "", 1, Long.MAX_VALUE, true);
-
- response = client.listReservations(request);
- Assert.assertNotNull(response);
- Assert.assertEquals(1, response.getReservationAllocationState().size());
- Assert.assertEquals(response.getReservationAllocationState().get(0)
- .getReservationId().getId(), reservationID.getId());
- // Verify that the full resource allocations exist.
- Assert.assertTrue(response.getReservationAllocationState().get(0)
- .getResourceAllocationRequests().size() > 0);
-
- // Verify that the full RDL is returned.
- ReservationRequests reservationRequests =
- response.getReservationAllocationState().get(0)
- .getReservationDefinition().getReservationRequests();
- Assert.assertEquals("R_ALL",
- reservationRequests.getInterpreter().toString());
- Assert.assertTrue(reservationRequests.getReservationResources().get(0)
- .getDuration() == duration);
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- @Test
- public void testListReservationsByInvalidTimeInterval() throws Exception {
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- // List reservations, search by invalid end time == -1.
- ReservationListRequest request = ReservationListRequest
- .newInstance(ReservationSystemTestUtil.reservationQ, "", 1, -1, true);
-
- ReservationListResponse response = client.listReservations(request);
- Assert.assertNotNull(response);
- Assert.assertEquals(1, response.getReservationAllocationState().size());
- Assert.assertEquals(response.getReservationAllocationState().get(0)
- .getReservationId().getId(), sRequest.getReservationId().getId());
-
- // List reservations, search by invalid end time < -1.
- request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, "", 1, -10, true);
-
- response = client.listReservations(request);
- Assert.assertNotNull(response);
- Assert.assertEquals(1, response.getReservationAllocationState().size());
- Assert.assertEquals(response.getReservationAllocationState().get(0)
- .getReservationId().getId(), sRequest.getReservationId().getId());
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- @Test
- public void testListReservationsByTimeIntervalContainingNoReservations()
- throws Exception {
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- // List reservations, search by very large start time.
- ReservationListRequest request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, "", Long.MAX_VALUE, -1,
- false);
-
- ReservationListResponse response = client.listReservations(request);
-
- // Ensure all reservations are filtered out.
- Assert.assertNotNull(response);
- Assert.assertEquals(response.getReservationAllocationState().size(), 0);
-
- duration = 30000;
- deadline = sRequest.getReservationDefinition().getDeadline();
-
- // List reservations, search by start time after the reservation
- // end time.
- request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, "", deadline + duration,
- deadline + 2 * duration, false);
-
- response = client.listReservations(request);
-
- // Ensure all reservations are filtered out.
- Assert.assertNotNull(response);
- Assert.assertEquals(response.getReservationAllocationState().size(), 0);
-
- arrival = clock.getTime();
- // List reservations, search by end time before the reservation start
- // time.
- request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, "", 0, arrival - duration,
- false);
-
- response = client.listReservations(request);
-
- // Ensure all reservations are filtered out.
- Assert.assertNotNull(response);
- Assert.assertEquals(response.getReservationAllocationState().size(), 0);
-
- // List reservations, search by very small end time.
- request = ReservationListRequest
- .newInstance(ReservationSystemTestUtil.reservationQ, "", 0, 1, false);
-
- response = client.listReservations(request);
-
- // Ensure all reservations are filtered out.
- Assert.assertNotNull(response);
- Assert.assertEquals(response.getReservationAllocationState().size(), 0);
-
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- @Test
- public void testReservationDelete() throws Exception {
- MiniYARNCluster cluster = setupMiniYARNCluster();
- YarnClient client = setupYarnClient(cluster);
- try {
- Clock clock = new UTCClock();
- long arrival = clock.getTime();
- long duration = 60000;
- long deadline = (long) (arrival + 1.05 * duration);
- ReservationSubmissionRequest sRequest =
- submitReservationTestHelper(client, arrival, deadline, duration);
-
- ReservationId reservationID = sRequest.getReservationId();
- // Delete the reservation
- ReservationDeleteRequest dRequest =
- ReservationDeleteRequest.newInstance(reservationID);
- ReservationDeleteResponse dResponse = client.deleteReservation(dRequest);
- Assert.assertNotNull(dResponse);
- System.out.println("Delete reservation response: " + dResponse);
-
- // List reservations, search by non-existent reservationID
- ReservationListRequest request = ReservationListRequest.newInstance(
- ReservationSystemTestUtil.reservationQ, reservationID.toString(), -1,
- -1, false);
-
- ReservationListResponse response = client.listReservations(request);
- Assert.assertNotNull(response);
- Assert.assertEquals(0, response.getReservationAllocationState().size());
- } finally {
- // clean-up
- if (client != null) {
- client.stop();
- }
- cluster.stop();
- }
- }
-
- private ReservationSubmissionRequest createSimpleReservationRequest(
- ReservationId reservationId, int numContainers, long arrival,
- long deadline, long duration) {
- // create a request with a single atomic ask
- ReservationRequest r =
- ReservationRequest.newInstance(Resource.newInstance(1024, 1),
- numContainers, 1, duration);
- ReservationRequests reqs =
- ReservationRequests.newInstance(Collections.singletonList(r),
- ReservationRequestInterpreter.R_ALL);
- ReservationDefinition rDef =
- ReservationDefinition.newInstance(arrival, deadline, reqs,
- "testYarnClient#reservation");
- ReservationSubmissionRequest request =
- ReservationSubmissionRequest.newInstance(rDef,
- ReservationSystemTestUtil.reservationQ, reservationId);
- return request;
- }
-
@Test(timeout = 30000, expected = ApplicationNotFoundException.class)
public void testShouldNotRetryForeverForNonNetworkExceptions() throws Exception {
- YarnConfiguration conf = new YarnConfiguration();
+ YarnConfiguration conf = getConf();
conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
ResourceManager rm = null;
@@ -1715,10 +1321,9 @@ public class TestYarnClient {
@Test
public void testSignalContainer() throws Exception {
- Configuration conf = new Configuration();
@SuppressWarnings("resource")
final YarnClient client = new MockYarnClient();
- client.init(conf);
+ client.init(getConf());
client.start();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
@@ -1741,7 +1346,7 @@ public class TestYarnClient {
boolean timelineClientBestEffort,
Throwable mockErr,
CreateTimelineClientErrorVerifier errVerifier) throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConf();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
timelineServiceEnabled);
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28d0fcbe/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
new file mode 100644
index 0000000..0836b7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
@@ -0,0 +1,521 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.client.api.impl;
+
+import com.google.common.base.Supplier;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationDeleteResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationListResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationSubmissionResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.ReservationUpdateResponse;
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.UTCClock;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.stream.Collectors;
+
+
+/**
+ * This class is to test class {@link YarnClient) and {@link YarnClientImpl}
+ * with Reservation.
+ */
+@RunWith(Parameterized.class)
+public class TestYarnClientWithReservation {
+ protected final static String TEST_DIR =
+ new File(System.getProperty("test.build.data", "/tmp")).getAbsolutePath();
+ protected final static String FS_ALLOC_FILE =
+ new File(TEST_DIR, "test-fs-queues.xml").getAbsolutePath();
+
+ public enum SchedulerType {
+ CAPACITY, FAIR
+ }
+
+ private SchedulerType schedulerType;
+
+ @Parameterized.Parameters(name = "{0}")
+ public static Collection<Object[]> getParameters() {
+ return Arrays.stream(SchedulerType.values()).map(
+ type -> new Object[]{type}).collect(Collectors.toList());
+ }
+
+ public TestYarnClientWithReservation(SchedulerType scheduler) {
+ this.schedulerType = scheduler;
+ }
+
+
+ private MiniYARNCluster setupMiniYARNCluster() throws Exception {
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
+
+ cluster.init(getConfigurationForReservation());
+ cluster.start();
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return cluster.getResourceManager().getRMContext()
+ .getReservationSystem()
+ .getPlan(ReservationSystemTestUtil.reservationQ)
+ .getTotalCapacity().getMemorySize() > 6000;
+ }
+ }, 10, 10000);
+
+ return cluster;
+ }
+
+ private Configuration getConfigurationForReservation() {
+ Configuration conf = new Configuration();
+ if (schedulerType == SchedulerType.FAIR) {
+ conf = configureReservationForFairScheduler();
+ conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
+ } else if (schedulerType == SchedulerType.CAPACITY) {
+ conf = configureReservationForCapacityScheduler();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getName());
+ }
+
+ conf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
+ return conf;
+ }
+
+ private Configuration configureReservationForCapacityScheduler() {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ return conf;
+ }
+
+ private Configuration configureReservationForFairScheduler() {
+ Configuration conf = new Configuration();
+ try {
+ PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"root\">");
+ out.println(" <queue name=\"default\"></queue>");
+ out.println(" <queue name=\"dedicated\">");
+ out.println(" <reservation></reservation>");
+ // set weight to 10 to make sure this queue get enough steady fair share
+ out.println(" <weight>10</weight>");
+ out.println(" </queue>");
+ out.println("</queue>");
+ out.println("<defaultQueueSchedulingPolicy>drf" +
+ "</defaultQueueSchedulingPolicy>");
+ out.println("</allocations>");
+ out.close();
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
+ return conf;
+ }
+
+ private YarnClient setupYarnClient(MiniYARNCluster cluster) {
+ final Configuration yarnConf = cluster.getConfig();
+ YarnClient client = YarnClient.createYarnClient();
+ client.init(yarnConf);
+ client.start();
+ return client;
+ }
+
+ private ReservationSubmissionRequest submitReservationTestHelper(
+ YarnClient client, long arrival, long deadline, long duration)
+ throws IOException, YarnException {
+ ReservationId reservationID = client.createReservation().getReservationId();
+ ReservationSubmissionRequest sRequest = createSimpleReservationRequest(
+ reservationID, 4, arrival, deadline, duration);
+ ReservationSubmissionResponse sResponse =
+ client.submitReservation(sRequest);
+ Assert.assertNotNull(sResponse);
+ Assert.assertNotNull(reservationID);
+ System.out.println("Submit reservation response: " + reservationID);
+
+ return sRequest;
+ }
+
+ @Before
+ public void setup() {
+ QueueMetrics.clearQueueMetrics();
+ DefaultMetricsSystem.setMiniClusterMode(true);
+ }
+
+ @Test
+ public void testCreateReservation() throws Exception {
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ // Submit the reservation again with the same request and make sure it
+ // passes.
+ client.submitReservation(sRequest);
+
+ // Submit the reservation with the same reservation id but different
+ // reservation definition, and ensure YarnException is thrown.
+ arrival = clock.getTime();
+ ReservationDefinition rDef = sRequest.getReservationDefinition();
+ rDef.setArrival(arrival + duration);
+ sRequest.setReservationDefinition(rDef);
+ try {
+ client.submitReservation(sRequest);
+ Assert.fail("Reservation submission should fail if a duplicate "
+ + "reservation id is used, but the reservation definition has been "
+ + "updated.");
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof YarnException);
+ }
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ @Test
+ public void testUpdateReservation() throws Exception {
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ ReservationDefinition rDef = sRequest.getReservationDefinition();
+ ReservationRequest rr =
+ rDef.getReservationRequests().getReservationResources().get(0);
+ ReservationId reservationID = sRequest.getReservationId();
+ rr.setNumContainers(5);
+ arrival = clock.getTime();
+ duration = 30000;
+ deadline = (long) (arrival + 1.05 * duration);
+ rr.setDuration(duration);
+ rDef.setArrival(arrival);
+ rDef.setDeadline(deadline);
+ ReservationUpdateRequest uRequest =
+ ReservationUpdateRequest.newInstance(rDef, reservationID);
+ ReservationUpdateResponse uResponse = client.updateReservation(uRequest);
+ Assert.assertNotNull(uResponse);
+ System.out.println("Update reservation response: " + uResponse);
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ private ReservationSubmissionRequest createSimpleReservationRequest(
+ ReservationId reservationId, int numContainers, long arrival,
+ long deadline, long duration) {
+ // create a request with a single atomic ask
+ ReservationRequest r =
+ ReservationRequest.newInstance(Resource.newInstance(1024, 1),
+ numContainers, 1, duration);
+ ReservationRequests reqs =
+ ReservationRequests.newInstance(Collections.singletonList(r),
+ ReservationRequestInterpreter.R_ALL);
+ ReservationDefinition rDef =
+ ReservationDefinition.newInstance(arrival, deadline, reqs,
+ "testYarnClient#reservation");
+ ReservationSubmissionRequest request =
+ ReservationSubmissionRequest.newInstance(rDef,
+ ReservationSystemTestUtil.reservationQ, reservationId);
+ return request;
+ }
+
+
+ @Test
+ public void testListReservationsByReservationId() throws Exception{
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ ReservationId reservationID = sRequest.getReservationId();
+ ReservationListRequest request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, reservationID.toString(), -1,
+ -1, false);
+ ReservationListResponse response = client.listReservations(request);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(1, response.getReservationAllocationState().size());
+ Assert.assertEquals(response.getReservationAllocationState().get(0)
+ .getReservationId().getId(), reservationID.getId());
+ Assert.assertEquals(response.getReservationAllocationState().get(0)
+ .getResourceAllocationRequests().size(), 0);
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ @Test
+ public void testListReservationsByTimeInterval() throws Exception {
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ // List reservations, search by a point in time within the reservation
+ // range.
+ arrival = clock.getTime();
+ ReservationId reservationID = sRequest.getReservationId();
+ ReservationListRequest request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, "", arrival + duration / 2,
+ arrival + duration / 2, true);
+
+ ReservationListResponse response = client.listReservations(request);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(1, response.getReservationAllocationState().size());
+ Assert.assertEquals(response.getReservationAllocationState().get(0)
+ .getReservationId().getId(), reservationID.getId());
+ // List reservations, search by time within reservation interval.
+ request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, "", 1, Long.MAX_VALUE, true);
+
+ response = client.listReservations(request);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(1, response.getReservationAllocationState().size());
+ Assert.assertEquals(response.getReservationAllocationState().get(0)
+ .getReservationId().getId(), reservationID.getId());
+ // Verify that the full resource allocations exist.
+ Assert.assertTrue(response.getReservationAllocationState().get(0)
+ .getResourceAllocationRequests().size() > 0);
+
+ // Verify that the full RDL is returned.
+ ReservationRequests reservationRequests =
+ response.getReservationAllocationState().get(0)
+ .getReservationDefinition().getReservationRequests();
+ Assert.assertEquals("R_ALL",
+ reservationRequests.getInterpreter().toString());
+ Assert.assertTrue(reservationRequests.getReservationResources().get(0)
+ .getDuration() == duration);
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ @Test
+ public void testListReservationsByInvalidTimeInterval() throws Exception {
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ // List reservations, search by invalid end time == -1.
+ ReservationListRequest request = ReservationListRequest
+ .newInstance(ReservationSystemTestUtil.reservationQ, "", 1, -1, true);
+
+ ReservationListResponse response = client.listReservations(request);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(1, response.getReservationAllocationState().size());
+ Assert.assertEquals(response.getReservationAllocationState().get(0)
+ .getReservationId().getId(), sRequest.getReservationId().getId());
+
+ // List reservations, search by invalid end time < -1.
+ request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, "", 1, -10, true);
+
+ response = client.listReservations(request);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(1, response.getReservationAllocationState().size());
+ Assert.assertEquals(response.getReservationAllocationState().get(0)
+ .getReservationId().getId(), sRequest.getReservationId().getId());
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ @Test
+ public void testListReservationsByTimeIntervalContainingNoReservations()
+ throws Exception {
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ // List reservations, search by very large start time.
+ ReservationListRequest request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, "", Long.MAX_VALUE, -1,
+ false);
+
+ ReservationListResponse response = client.listReservations(request);
+
+ // Ensure all reservations are filtered out.
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.getReservationAllocationState().size(), 0);
+
+ duration = 30000;
+ deadline = sRequest.getReservationDefinition().getDeadline();
+
+ // List reservations, search by start time after the reservation
+ // end time.
+ request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, "", deadline + duration,
+ deadline + 2 * duration, false);
+
+ response = client.listReservations(request);
+
+ // Ensure all reservations are filtered out.
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.getReservationAllocationState().size(), 0);
+
+ arrival = clock.getTime();
+ // List reservations, search by end time before the reservation start
+ // time.
+ request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, "", 0, arrival - duration,
+ false);
+
+ response = client.listReservations(request);
+
+ // Ensure all reservations are filtered out.
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.getReservationAllocationState().size(), 0);
+
+ // List reservations, search by very small end time.
+ request = ReservationListRequest
+ .newInstance(ReservationSystemTestUtil.reservationQ, "", 0, 1, false);
+
+ response = client.listReservations(request);
+
+ // Ensure all reservations are filtered out.
+ Assert.assertNotNull(response);
+ Assert.assertEquals(response.getReservationAllocationState().size(), 0);
+
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+ @Test
+ public void testReservationDelete() throws Exception {
+ MiniYARNCluster cluster = setupMiniYARNCluster();
+ YarnClient client = setupYarnClient(cluster);
+ try {
+ Clock clock = new UTCClock();
+ long arrival = clock.getTime();
+ long duration = 60000;
+ long deadline = (long) (arrival + 1.05 * duration);
+ ReservationSubmissionRequest sRequest =
+ submitReservationTestHelper(client, arrival, deadline, duration);
+
+ ReservationId reservationID = sRequest.getReservationId();
+ // Delete the reservation
+ ReservationDeleteRequest dRequest =
+ ReservationDeleteRequest.newInstance(reservationID);
+ ReservationDeleteResponse dResponse = client.deleteReservation(dRequest);
+ Assert.assertNotNull(dResponse);
+ System.out.println("Delete reservation response: " + dResponse);
+
+ // List reservations, search by non-existent reservationID
+ ReservationListRequest request = ReservationListRequest.newInstance(
+ ReservationSystemTestUtil.reservationQ, reservationID.toString(), -1,
+ -1, false);
+
+ ReservationListResponse response = client.listReservations(request);
+ Assert.assertNotNull(response);
+ Assert.assertEquals(0, response.getReservationAllocationState().size());
+ } finally {
+ // clean-up
+ if (client != null) {
+ client.stop();
+ }
+ cluster.stop();
+ }
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/19] hadoop git commit: YARN-7469. Capacity Scheduler Intra-queue
preemption: User can starve if newest app is exactly at user limit.
Contributed by Eric Payne.
Posted by as...@apache.org.
YARN-7469. Capacity Scheduler Intra-queue preemption: User can starve if newest app is exactly at user limit. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61ace174
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61ace174
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61ace174
Branch: refs/heads/YARN-6592
Commit: 61ace174cdcbca9d22abce7aa0aa71148f37ad55
Parents: f4d5d20
Author: Sunil G <su...@apache.org>
Authored: Thu Nov 16 22:34:23 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Nov 16 22:34:23 2017 +0530
----------------------------------------------------------------------
.../FifoIntraQueuePreemptionPlugin.java | 6 ++++
...alCapacityPreemptionPolicyMockFramework.java | 3 ++
...cityPreemptionPolicyIntraQueueUserLimit.java | 35 ++++++++++++++++++++
3 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ace174/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
index 00ae3da..3332f2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoIntraQueuePreemptionPlugin.java
@@ -203,6 +203,12 @@ public class FifoIntraQueuePreemptionPlugin
Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.selected);
Resources.subtractFromNonNegative(preemtableFromApp, tmpApp.getAMUsed());
+ if (context.getIntraQueuePreemptionOrderPolicy()
+ .equals(IntraQueuePreemptionOrderPolicy.USERLIMIT_FIRST)) {
+ Resources.subtractFromNonNegative(preemtableFromApp,
+ tmpApp.getFiCaSchedulerApp().getCSLeafQueue().getMinimumAllocation());
+ }
+
// Calculate toBePreempted from apps as follows:
// app.preemptable = min(max(app.used - app.selected - app.ideal, 0),
// intra_q_preemptable)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ace174/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
index 4fc0ea4..0bc5cb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java
@@ -358,6 +358,9 @@ public class ProportionalCapacityPreemptionPolicyMockFramework {
queue = (LeafQueue) nameToCSQueues.get(queueName);
queue.getApplications().add(app);
queue.getAllApplications().add(app);
+ when(queue.getMinimumAllocation())
+ .thenReturn(Resource.newInstance(1,1));
+ when(app.getCSLeafQueue()).thenReturn(queue);
HashSet<String> users = userMap.get(queueName);
if (null == users) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61ace174/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
index 7df52f9..0440db3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit.java
@@ -896,4 +896,39 @@ public class TestProportionalCapacityPreemptionPolicyIntraQueueUserLimit
new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
getAppAttemptId(1))));
}
+
+ @Test
+ public void testSimpleIntraQueuePreemptionOneUserUnderOneUserAtOneUserAbove()
+ throws IOException {
+ conf.setFloat(CapacitySchedulerConfiguration.
+ INTRAQUEUE_PREEMPTION_MAX_ALLOWABLE_LIMIT,
+ (float) 0.5);
+
+ String labelsConfig = "=100,true;";
+ String nodesConfig = // n1 has no label
+ "n1= res=100";
+ String queuesConfig =
+ // guaranteed,max,used,pending,reserved
+ "root(=[100 100 100 1 0]);" + // root
+ "-a(=[100 100 100 1 0])"; // a
+
+ String appsConfig =
+ // queueName\t(priority,resource,host,expression,#repeat,reserved,pending)
+ "a\t" // app1 in a
+ + "(1,1,n1,,65,false,0,user1);" +
+ "a\t" // app2 in a
+ + "(1,1,n1,,35,false,0,user2);" +
+ "a\t" // app3 in a
+ + "(1,1,n1,,0,false,1,user3)"
+ ;
+
+ buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig);
+ policy.editSchedule();
+
+ // app2 is right at its user limit and app1 needs one resource. Should
+ // preempt 1 container.
+ verify(mDisp, times(1)).handle(argThat(
+ new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor(
+ getAppAttemptId(1))));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/19] hadoop git commit: Revert "HDFS-12681. Fold
HdfsLocatedFileStatus into HdfsFileStatus."
Posted by as...@apache.org.
Revert "HDFS-12681. Fold HdfsLocatedFileStatus into HdfsFileStatus."
This reverts commit b85603e3f85e85da406241b991f3a9974384c3aa.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/675e9a8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/675e9a8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/675e9a8f
Branch: refs/heads/YARN-6592
Commit: 675e9a8f57570771a0219d95940681b067d36b94
Parents: b1941b2
Author: Chris Douglas <cd...@apache.org>
Authored: Wed Nov 15 19:17:46 2017 -0800
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed Nov 15 19:20:37 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/fs/LocatedFileStatus.java | 11 +-
.../main/java/org/apache/hadoop/fs/Hdfs.java | 4 +-
.../hadoop/hdfs/DistributedFileSystem.java | 4 +-
.../hadoop/hdfs/protocol/HdfsFileStatus.java | 80 ++++----------
.../hdfs/protocol/HdfsLocatedFileStatus.java | 110 +++++++++++++++++++
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 65 +++++------
.../dev-support/findbugsExcludeFile.xml | 7 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 6 +-
.../server/namenode/FSDirStatAndListingOp.java | 61 +++++-----
.../hadoop/hdfs/TestBlockStoragePolicy.java | 10 +-
.../apache/hadoop/hdfs/TestDFSOutputStream.java | 2 +-
.../hdfs/server/mover/TestStorageMover.java | 8 +-
12 files changed, 217 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
index 9cc81d3..29e1998 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocatedFileStatus.java
@@ -135,16 +135,7 @@ public class LocatedFileStatus extends FileStatus {
public BlockLocation[] getBlockLocations() {
return locations;
}
-
- /**
- * Hook for subclasses to lazily set block locations. The {@link #locations}
- * field should be null before this is called.
- * @param locations Block locations for this instance.
- */
- protected void setBlockLocations(BlockLocation[] locations) {
- this.locations = locations;
- }
-
+
/**
* Compare this FileStatus to another FileStatus
* @param o the FileStatus to be compared.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
index f306d06..0138195 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/fs/Hdfs.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.AccessControlException;
@@ -187,7 +188,8 @@ public class Hdfs extends AbstractFileSystem {
@Override
public LocatedFileStatus next() throws IOException {
- return getNext().makeQualifiedLocated(getUri(), p);
+ return ((HdfsLocatedFileStatus)getNext()).makeQualifiedLocated(
+ getUri(), p);
}
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index eef83d7..9db12e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -85,6 +85,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
@@ -1211,7 +1212,8 @@ public class DistributedFileSystem extends FileSystem {
T next;
HdfsFileStatus fileStat = thisListing.getPartialListing()[i++];
if (needLocation) {
- next = (T)fileStat.makeQualifiedLocated(getUri(), p);
+ next = (T)((HdfsLocatedFileStatus)fileStat)
+ .makeQualifiedLocated(getUri(), p);
} else {
next = (T)fileStat.makeQualified(getUri(), p);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
index fce2c44..0499f2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java
@@ -26,17 +26,15 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
-/**
- * HDFS metadata for an entity in the filesystem.
+/** Interface that represents the over the wire information for a file.
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
-public final class HdfsFileStatus extends LocatedFileStatus {
+public class HdfsFileStatus extends FileStatus {
private static final long serialVersionUID = 0x126eb82a;
@@ -51,12 +49,11 @@ public final class HdfsFileStatus extends LocatedFileStatus {
private final int childrenNum;
private final byte storagePolicy;
- // BlockLocations[] is the user-facing type
- private transient LocatedBlocks hdfsloc;
-
public static final byte[] EMPTY_NAME = new byte[0];
- /** Set of features potentially active on an instance. */
+ /**
+ * Set of features potentially active on an instance.
+ */
public enum Flags {
HAS_ACL,
HAS_CRYPT,
@@ -84,19 +81,18 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* @param storagePolicy ID which specifies storage policy
* @param ecPolicy the erasure coding policy
*/
- private HdfsFileStatus(long length, boolean isdir, int replication,
+ protected HdfsFileStatus(long length, boolean isdir, int replication,
long blocksize, long mtime, long atime,
FsPermission permission, EnumSet<Flags> flags,
String owner, String group,
byte[] symlink, byte[] path, long fileId,
int childrenNum, FileEncryptionInfo feInfo,
- byte storagePolicy, ErasureCodingPolicy ecPolicy,
- LocatedBlocks hdfsloc) {
+ byte storagePolicy, ErasureCodingPolicy ecPolicy) {
super(length, isdir, replication, blocksize, mtime,
atime, convert(isdir, symlink != null, permission, flags),
owner, group, null, null,
flags.contains(Flags.HAS_ACL), flags.contains(Flags.HAS_CRYPT),
- flags.contains(Flags.HAS_EC), null);
+ flags.contains(Flags.HAS_EC));
this.flags = flags;
this.uSymlink = symlink;
this.uPath = path;
@@ -105,7 +101,6 @@ public final class HdfsFileStatus extends LocatedFileStatus {
this.feInfo = feInfo;
this.storagePolicy = storagePolicy;
this.ecPolicy = ecPolicy;
- this.hdfsloc = hdfsloc;
}
/**
@@ -157,7 +152,7 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* Check if the local name is empty.
* @return true if the name is empty
*/
- public boolean isEmptyLocalName() {
+ public final boolean isEmptyLocalName() {
return uPath.length == 0;
}
@@ -165,7 +160,7 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* Get the string representation of the local name.
* @return the local name in string
*/
- public String getLocalName() {
+ public final String getLocalName() {
return DFSUtilClient.bytes2String(uPath);
}
@@ -173,7 +168,7 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* Get the Java UTF8 representation of the local name.
* @return the local name in java UTF8
*/
- public byte[] getLocalNameInBytes() {
+ public final byte[] getLocalNameInBytes() {
return uPath;
}
@@ -182,7 +177,7 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* @param parent the parent path
* @return the full path in string
*/
- public String getFullName(String parent) {
+ public final String getFullName(final String parent) {
if (isEmptyLocalName()) {
return parent;
}
@@ -200,7 +195,7 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* @param parent the parent path
* @return the full path
*/
- public Path getFullPath(Path parent) {
+ public final Path getFullPath(final Path parent) {
if (isEmptyLocalName()) {
return parent;
}
@@ -224,15 +219,15 @@ public final class HdfsFileStatus extends LocatedFileStatus {
/**
* Opaque referant for the symlink, to be resolved at the client.
*/
- public byte[] getSymlinkInBytes() {
+ public final byte[] getSymlinkInBytes() {
return uSymlink;
}
- public long getFileId() {
+ public final long getFileId() {
return fileId;
}
- public FileEncryptionInfo getFileEncryptionInfo() {
+ public final FileEncryptionInfo getFileEncryptionInfo() {
return feInfo;
}
@@ -244,12 +239,12 @@ public final class HdfsFileStatus extends LocatedFileStatus {
return ecPolicy;
}
- public int getChildrenNum() {
+ public final int getChildrenNum() {
return childrenNum;
}
/** @return the storage policy id */
- public byte getStoragePolicy() {
+ public final byte getStoragePolicy() {
return storagePolicy;
}
@@ -262,10 +257,6 @@ public final class HdfsFileStatus extends LocatedFileStatus {
return flags.contains(Flags.SNAPSHOT_ENABLED);
}
- public LocatedBlocks getLocatedBlocks() {
- return hdfsloc;
- }
-
@Override
public boolean equals(Object o) {
// satisfy findbugs
@@ -286,30 +277,11 @@ public final class HdfsFileStatus extends LocatedFileStatus {
* @param parent Parent path of this element.
* @return Reference to this instance.
*/
- public FileStatus makeQualified(URI defaultUri, Path parent) {
+ public final FileStatus makeQualified(URI defaultUri, Path parent) {
// fully-qualify path
setPath(getFullPath(parent).makeQualified(defaultUri, null));
return this; // API compatibility
- }
- /**
- * This function is used to transform the underlying HDFS LocatedBlocks to
- * BlockLocations. This method must be invoked before
- * {@link #getBlockLocations()}.
- *
- * The returned BlockLocation will have different formats for replicated
- * and erasure coded file.
- * Please refer to
- * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations
- * (FileStatus, long, long)}
- * for examples.
- */
- public LocatedFileStatus makeQualifiedLocated(URI defaultUri,
- Path path) {
- makeQualified(defaultUri, path);
- setBlockLocations(
- DFSUtilClient.locatedBlocks2Locations(getLocatedBlocks()));
- return this;
}
/**
@@ -339,7 +311,6 @@ public final class HdfsFileStatus extends LocatedFileStatus {
private byte storagePolicy =
HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED;
private ErasureCodingPolicy ecPolicy = null;
- private LocatedBlocks locations = null;
/**
* Set the length of the entity (default = 0).
@@ -519,23 +490,12 @@ public final class HdfsFileStatus extends LocatedFileStatus {
}
/**
- * Set the block locations for this entity (default = null).
- * @param locations HDFS locations
- * (see {@link #makeQualifiedLocated(URI, Path)})
- * @return This Builder instance
- */
- public Builder locations(LocatedBlocks locations) {
- this.locations = locations;
- return this;
- }
-
- /**
* @return An {@link HdfsFileStatus} instance from these parameters.
*/
public HdfsFileStatus build() {
return new HdfsFileStatus(length, isdir, replication, blocksize,
mtime, atime, permission, flags, owner, group, symlink, path, fileId,
- childrenNum, feInfo, storagePolicy, ecPolicy, locations);
+ childrenNum, feInfo, storagePolicy, ecPolicy);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
new file mode 100644
index 0000000..193aae2
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import java.net.URI;
+import java.util.EnumSet;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSUtilClient;
+
+/**
+ * Interface that represents the over the wire information
+ * including block locations for a file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class HdfsLocatedFileStatus extends HdfsFileStatus {
+
+ private static final long serialVersionUID = 0x23c73328;
+
+ /**
+ * Left transient, because {@link #makeQualifiedLocated(URI,Path)}
+ * is the user-facing type.
+ */
+ private transient LocatedBlocks locations;
+
+ /**
+ * Constructor
+ *
+ * @param length size
+ * @param isdir if this is directory
+ * @param block_replication the file's replication factor
+ * @param blocksize the file's block size
+ * @param modification_time most recent modification time
+ * @param access_time most recent access time
+ * @param permission permission
+ * @param owner owner
+ * @param group group
+ * @param symlink symbolic link
+ * @param path local path name in java UTF8 format
+ * @param fileId the file id
+ * @param locations block locations
+ * @param feInfo file encryption info
+ */
+ public HdfsLocatedFileStatus(long length, boolean isdir,
+ int block_replication, long blocksize, long modification_time,
+ long access_time, FsPermission permission, EnumSet<Flags> flags,
+ String owner, String group, byte[] symlink, byte[] path, long fileId,
+ LocatedBlocks locations, int childrenNum, FileEncryptionInfo feInfo,
+ byte storagePolicy, ErasureCodingPolicy ecPolicy) {
+ super(length, isdir, block_replication, blocksize, modification_time,
+ access_time, permission, flags, owner, group, symlink, path, fileId,
+ childrenNum, feInfo, storagePolicy, ecPolicy);
+ this.locations = locations;
+ }
+
+ public LocatedBlocks getBlockLocations() {
+ return locations;
+ }
+
+ /**
+ * This function is used to transform the underlying HDFS LocatedBlocks to
+ * BlockLocations.
+ *
+ * The returned BlockLocation will have different formats for replicated
+ * and erasure coded file.
+ * Please refer to
+ * {@link org.apache.hadoop.fs.FileSystem#getFileBlockLocations
+ * (FileStatus, long, long)}
+ * for examples.
+ */
+ public final LocatedFileStatus makeQualifiedLocated(URI defaultUri,
+ Path path) {
+ makeQualified(defaultUri, path);
+ return new LocatedFileStatus(this,
+ DFSUtilClient.locatedBlocks2Locations(getBlockLocations()));
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ // satisfy findbugs
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ // satisfy findbugs
+ return super.hashCode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 54fab6d..ae3921c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsPathHandle;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -1584,36 +1585,23 @@ public class PBHelperClient {
EnumSet<HdfsFileStatus.Flags> flags = fs.hasFlags()
? convertFlags(fs.getFlags())
: convertFlags(fs.getPermission());
- return new HdfsFileStatus.Builder()
- .length(fs.getLength())
- .isdir(fs.getFileType().equals(FileType.IS_DIR))
- .replication(fs.getBlockReplication())
- .blocksize(fs.getBlocksize())
- .mtime(fs.getModificationTime())
- .atime(fs.getAccessTime())
- .perm(convert(fs.getPermission()))
- .flags(flags)
- .owner(fs.getOwner())
- .group(fs.getGroup())
- .symlink(FileType.IS_SYMLINK.equals(fs.getFileType())
- ? fs.getSymlink().toByteArray()
- : null)
- .path(fs.getPath().toByteArray())
- .fileId(fs.hasFileId()
- ? fs.getFileId()
- : HdfsConstants.GRANDFATHER_INODE_ID)
- .locations(fs.hasLocations() ? convert(fs.getLocations()) : null)
- .children(fs.hasChildrenNum() ? fs.getChildrenNum() : -1)
- .feInfo(fs.hasFileEncryptionInfo()
- ? convert(fs.getFileEncryptionInfo())
- : null)
- .storagePolicy(fs.hasStoragePolicy()
- ? (byte) fs.getStoragePolicy()
- : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED)
- .ecPolicy(fs.hasEcPolicy()
- ? convertErasureCodingPolicy(fs.getEcPolicy())
- : null)
- .build();
+ return new HdfsLocatedFileStatus(
+ fs.getLength(), fs.getFileType().equals(FileType.IS_DIR),
+ fs.getBlockReplication(), fs.getBlocksize(),
+ fs.getModificationTime(), fs.getAccessTime(),
+ convert(fs.getPermission()),
+ flags,
+ fs.getOwner(), fs.getGroup(),
+ fs.getFileType().equals(FileType.IS_SYMLINK) ?
+ fs.getSymlink().toByteArray() : null,
+ fs.getPath().toByteArray(),
+ fs.hasFileId()? fs.getFileId(): HdfsConstants.GRANDFATHER_INODE_ID,
+ fs.hasLocations() ? convert(fs.getLocations()) : null,
+ fs.hasChildrenNum() ? fs.getChildrenNum() : -1,
+ fs.hasFileEncryptionInfo() ? convert(fs.getFileEncryptionInfo()) : null,
+ fs.hasStoragePolicy() ? (byte) fs.getStoragePolicy()
+ : HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED,
+ fs.hasEcPolicy() ? convertErasureCodingPolicy(fs.getEcPolicy()) : null);
}
private static EnumSet<HdfsFileStatus.Flags> convertFlags(int flags) {
@@ -1876,10 +1864,10 @@ public class PBHelperClient {
if (dl == null)
return null;
List<HdfsFileStatusProto> partList = dl.getPartialListingList();
- return new DirectoryListing(partList.isEmpty()
- ? new HdfsFileStatus[0]
- : convert(partList.toArray(new HdfsFileStatusProto[partList.size()])),
- dl.getRemainingEntries());
+ return new DirectoryListing(partList.isEmpty() ?
+ new HdfsLocatedFileStatus[0] :
+ convert(partList.toArray(new HdfsFileStatusProto[partList.size()])),
+ dl.getRemainingEntries());
}
public static HdfsFileStatus[] convert(HdfsFileStatusProto[] fs) {
@@ -2173,9 +2161,12 @@ public class PBHelperClient {
if (fs.getFileEncryptionInfo() != null) {
builder.setFileEncryptionInfo(convert(fs.getFileEncryptionInfo()));
}
- LocatedBlocks locations = fs.getLocatedBlocks();
- if (locations != null) {
- builder.setLocations(convert(locations));
+ if (fs instanceof HdfsLocatedFileStatus) {
+ final HdfsLocatedFileStatus lfs = (HdfsLocatedFileStatus) fs;
+ LocatedBlocks locations = lfs.getBlockLocations();
+ if (locations != null) {
+ builder.setLocations(convert(locations));
+ }
}
if(fs.getErasureCodingPolicy() != null) {
builder.setEcPolicy(convertErasureCodingPolicy(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index 3f2814f..4b958b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -260,10 +260,11 @@
<Method name="visitFile" />
<Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE" />
</Match>
- <!-- BlockLocations are user-facing, but LocatedBlocks are not. -->
+ <!-- HdfsFileStatus is user-facing, but HdfsLocatedFileStatus is not.
+ Defensible compatibility choices over time create odd corners. -->
<Match>
- <Class name="org.apache.hadoop.hdfs.protocol.HdfsFileStatus" />
- <Field name="hdfsloc" />
+ <Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" />
+ <Field name="locations" />
<Bug pattern="SE_TRANSIENT_FIELD_NOT_RESTORED" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index aa22978..b653f4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -364,7 +364,7 @@ public class Mover {
if (!isSnapshotPathInCurrent(fullPath)) {
// the full path is a snapshot path but it is also included in the
// current directory tree, thus ignore it.
- processFile(fullPath, status, result);
+ processFile(fullPath, (HdfsLocatedFileStatus) status, result);
}
} catch (IOException e) {
LOG.warn("Failed to check the status of " + parent
@@ -374,7 +374,7 @@ public class Mover {
}
/** @return true if it is necessary to run another round of migration */
- private void processFile(String fullPath, HdfsFileStatus status,
+ private void processFile(String fullPath, HdfsLocatedFileStatus status,
Result result) {
byte policyId = status.getStoragePolicy();
if (policyId == HdfsConstants.BLOCK_STORAGE_POLICY_ID_UNSPECIFIED) {
@@ -395,7 +395,7 @@ public class Mover {
status.getReplication());
final ErasureCodingPolicy ecPolicy = status.getErasureCodingPolicy();
- final LocatedBlocks locatedBlocks = status.getLocatedBlocks();
+ final LocatedBlocks locatedBlocks = status.getBlockLocations();
final boolean lastBlkComplete = locatedBlocks.isLastBlockComplete();
List<LocatedBlock> lbs = locatedBlocks.getLocatedBlocks();
for (int i = 0; i < lbs.size(); i++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
index f1e2660..b6e38a7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirStatAndListingOp.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
@@ -255,13 +256,14 @@ class FSDirStatAndListingOp {
listing[i] =
createFileStatus(fsd, iip, child, childStoragePolicy, needLocation);
listingCnt++;
- LocatedBlocks blks = listing[i].getLocatedBlocks();
- if (blks != null) {
- // Once we hit lsLimit locations, stop.
- // This helps to prevent excessively large response payloads.
- // Approximate #locations with locatedBlockCount() * repl_factor
- locationBudget -=
- blks.locatedBlockCount() * listing[i].getReplication();
+ if (listing[i] instanceof HdfsLocatedFileStatus) {
+ // Once we hit lsLimit locations, stop.
+ // This helps to prevent excessively large response payloads.
+ // Approximate #locations with locatedBlockCount() * repl_factor
+ LocatedBlocks blks =
+ ((HdfsLocatedFileStatus)listing[i]).getBlockLocations();
+ locationBudget -= (blks == null) ? 0 :
+ blks.locatedBlockCount() * listing[i].getReplication();
}
}
// truncate return array if necessary
@@ -484,26 +486,31 @@ class FSDirStatAndListingOp {
String owner, String group, byte[] symlink, byte[] path, long fileId,
int childrenNum, FileEncryptionInfo feInfo, byte storagePolicy,
ErasureCodingPolicy ecPolicy, LocatedBlocks locations) {
- return new HdfsFileStatus.Builder()
- .length(length)
- .isdir(isdir)
- .replication(replication)
- .blocksize(blocksize)
- .mtime(mtime)
- .atime(atime)
- .perm(permission)
- .flags(flags)
- .owner(owner)
- .group(group)
- .symlink(symlink)
- .path(path)
- .fileId(fileId)
- .children(childrenNum)
- .feInfo(feInfo)
- .storagePolicy(storagePolicy)
- .ecPolicy(ecPolicy)
- .locations(locations)
- .build();
+ if (locations == null) {
+ return new HdfsFileStatus.Builder()
+ .length(length)
+ .isdir(isdir)
+ .replication(replication)
+ .blocksize(blocksize)
+ .mtime(mtime)
+ .atime(atime)
+ .perm(permission)
+ .flags(flags)
+ .owner(owner)
+ .group(group)
+ .symlink(symlink)
+ .path(path)
+ .fileId(fileId)
+ .children(childrenNum)
+ .feInfo(feInfo)
+ .storagePolicy(storagePolicy)
+ .ecPolicy(ecPolicy)
+ .build();
+ } else {
+ return new HdfsLocatedFileStatus(length, isdir, replication, blocksize,
+ mtime, atime, permission, flags, owner, group, symlink, path,
+ fileId, locations, childrenNum, feInfo, storagePolicy, ecPolicy);
+ }
}
private static ContentSummary getContentSummaryInt(FSDirectory fsd,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index bc060e1..3a8fb59 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -1092,11 +1092,11 @@ public class TestBlockStoragePolicy {
return types;
}
- private void checkLocatedBlocks(HdfsFileStatus status, int blockNum,
+ private void checkLocatedBlocks(HdfsLocatedFileStatus status, int blockNum,
int replicaNum, StorageType... types) {
List<StorageType> typeList = Lists.newArrayList();
Collections.addAll(typeList, types);
- LocatedBlocks lbs = status.getLocatedBlocks();
+ LocatedBlocks lbs = status.getBlockLocations();
Assert.assertEquals(blockNum, lbs.getLocatedBlocks().size());
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
Assert.assertEquals(replicaNum, lb.getStorageTypes().length);
@@ -1127,7 +1127,7 @@ public class TestBlockStoragePolicy {
HdfsFileStatus[] status = fs.getClient().listPaths(foo.toString(),
HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
checkDirectoryListing(status, policyId);
- HdfsFileStatus fooStatus = status[0];
+ HdfsLocatedFileStatus fooStatus = (HdfsLocatedFileStatus) status[0];
checkLocatedBlocks(fooStatus, 1, 3, before);
// change the replication factor to 5
@@ -1140,7 +1140,7 @@ public class TestBlockStoragePolicy {
status = fs.getClient().listPaths(foo.toString(),
HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
checkDirectoryListing(status, policyId);
- fooStatus = status[0];
+ fooStatus = (HdfsLocatedFileStatus) status[0];
checkLocatedBlocks(fooStatus, 1, numDataNodes, after);
// change the replication factor back to 3
@@ -1157,7 +1157,7 @@ public class TestBlockStoragePolicy {
status = fs.getClient().listPaths(foo.toString(),
HdfsFileStatus.EMPTY_NAME, true).getPartialListing();
checkDirectoryListing(status, policyId);
- fooStatus = status[0];
+ fooStatus = (HdfsLocatedFileStatus) status[0];
checkLocatedBlocks(fooStatus, 1, REPLICATION, before);
} finally {
cluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
index 3f4acba..f281a3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSOutputStream.java
@@ -274,7 +274,7 @@ public class TestDFSOutputStream {
when(client.getTracer()).thenReturn(FsTracer.get(new Configuration()));
client.clientRunning = true;
DataStreamer stream = new DataStreamer(
- new HdfsFileStatus.Builder().build(),
+ mock(HdfsFileStatus.class),
mock(ExtendedBlock.class),
client,
"foo", null, null, null, null, null, null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675e9a8f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
index 0c31288..764a0db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestStorageMover.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtocol;
@@ -311,17 +312,18 @@ public class TestStorageMover {
private void verifyFile(final Path parent, final HdfsFileStatus status,
final Byte expectedPolicyId) throws Exception {
- byte policyId = status.getStoragePolicy();
+ HdfsLocatedFileStatus fileStatus = (HdfsLocatedFileStatus) status;
+ byte policyId = fileStatus.getStoragePolicy();
BlockStoragePolicy policy = policies.getPolicy(policyId);
if (expectedPolicyId != null) {
Assert.assertEquals((byte)expectedPolicyId, policy.getId());
}
final List<StorageType> types = policy.chooseStorageTypes(
status.getReplication());
- for(LocatedBlock lb : status.getLocatedBlocks().getLocatedBlocks()) {
+ for(LocatedBlock lb : fileStatus.getBlockLocations().getLocatedBlocks()) {
final Mover.StorageTypeDiff diff = new Mover.StorageTypeDiff(types,
lb.getStorageTypes());
- Assert.assertTrue(status.getFullName(parent.toString())
+ Assert.assertTrue(fileStatus.getFullName(parent.toString())
+ " with policy " + policy + " has non-empty overlap: " + diff
+ ", the corresponding block is " + lb.getBlock().getLocalBlock(),
diff.removeOverlap(true));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/19] hadoop git commit: YARN-7430. Enable user re-mapping for
Docker containers by default. Contributed by Eric Yang.
Posted by as...@apache.org.
YARN-7430. Enable user re-mapping for Docker containers by default. Contributed by Eric Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f0b238a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f0b238a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f0b238a
Branch: refs/heads/YARN-6592
Commit: 5f0b238a118f3992bd149d8c02e6a1376dee96d7
Parents: e182e77
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Nov 17 12:04:47 2017 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Nov 17 12:04:47 2017 +0530
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f0b238a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 4799137..34257ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1735,7 +1735,7 @@ public class YarnConfiguration extends Configuration {
DOCKER_CONTAINER_RUNTIME_PREFIX + "enable-userremapping.allowed";
/** Set enable user remapping as false by default. */
- public static final boolean DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING = false;
+ public static final boolean DEFAULT_NM_DOCKER_ENABLE_USER_REMAPPING = true;
/** lower limit for acceptable uids of user remapped user. */
public static final String NM_DOCKER_USER_REMAPPING_UID_THRESHOLD =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/19] hadoop git commit: YARN-7492. Set up SASS for new YARN UI
styling. Contributed by Vasudevan Skm.
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a13426/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
index dc45d7e..fb35ea7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn.lock
@@ -25,6 +25,15 @@ after@0.8.1:
version "0.8.1"
resolved "https://registry.yarnpkg.com/after/-/after-0.8.1.tgz#ab5d4fb883f596816d3515f8f791c0af486dd627"
+ajv@^5.1.0:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/ajv/-/ajv-5.3.0.tgz#4414ff74a50879c208ee5fdc826e32c303549eda"
+ dependencies:
+ co "^4.6.0"
+ fast-deep-equal "^1.0.0"
+ fast-json-stable-stringify "^2.0.0"
+ json-schema-traverse "^0.3.0"
+
align-text@^0.1.1, align-text@^0.1.3:
version "0.1.4"
resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@@ -94,6 +103,10 @@ anymatch@^1.3.0:
arrify "^1.0.0"
micromatch "^2.1.5"
+aproba@^1.0.3:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/aproba/-/aproba-1.2.0.tgz#6802e6264efd18c790a1b0d517f0f2627bf2c94a"
+
archy@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/archy/-/archy-1.0.0.tgz#f9c8c13757cc1dd7bc379ac77b2c62a5c2868c40"
@@ -132,6 +145,10 @@ array-equal@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93"
+array-find-index@^1.0.1:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/array-find-index/-/array-find-index-1.0.2.tgz#df010aa1287e164bbda6f9723b0a96a1ec4187a1"
+
array-flatten@1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/array-flatten/-/array-flatten-1.1.1.tgz#9a5f699051b1e7073328f2a008968b64ea2955d2"
@@ -202,6 +219,10 @@ async-disk-cache@^1.0.0:
rimraf "^2.5.3"
rsvp "^3.0.18"
+async-foreach@^0.1.3:
+ version "0.1.3"
+ resolved "https://registry.yarnpkg.com/async-foreach/-/async-foreach-0.1.3.tgz#36121f845c0578172de419a97dbeb1d16ec34542"
+
async-some@~1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/async-some/-/async-some-1.0.2.tgz#4d8a81620d5958791b5b98f802d3207776e95509"
@@ -230,11 +251,19 @@ async@~0.8.0:
version "0.8.0"
resolved "https://registry.yarnpkg.com/async/-/async-0.8.0.tgz#ee65ec77298c2ff1456bc4418a052d0f06435112"
+asynckit@^0.4.0:
+ version "0.4.0"
+ resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
+
aws-sign2@~0.6.0:
version "0.6.0"
resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.6.0.tgz#14342dd38dbcc94d0e5b87d763cd63612c0e794f"
-aws4@^1.2.1:
+aws-sign2@~0.7.0:
+ version "0.7.0"
+ resolved "https://registry.yarnpkg.com/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8"
+
+aws4@^1.2.1, aws4@^1.6.0:
version "1.6.0"
resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.6.0.tgz#83ef5ca860b2b32e4a0deedee8c771b9db57471e"
@@ -375,6 +404,10 @@ balanced-match@^0.4.1:
version "0.4.2"
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-0.4.2.tgz#cb3f3e3c732dc0f01ee70b403f302e61d7709838"
+balanced-match@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
+
base64-arraybuffer@0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-0.1.2.tgz#474df4a9f2da24e05df3158c3b1db3c3cd46a154"
@@ -473,6 +506,18 @@ boom@2.x.x:
dependencies:
hoek "2.x.x"
+boom@4.x.x:
+ version "4.3.1"
+ resolved "https://registry.yarnpkg.com/boom/-/boom-4.3.1.tgz#4f8a3005cb4a7e3889f749030fd25b96e01d2e31"
+ dependencies:
+ hoek "4.x.x"
+
+boom@5.x.x:
+ version "5.2.0"
+ resolved "https://registry.yarnpkg.com/boom/-/boom-5.2.0.tgz#5dd9da6ee3a5f302077436290cb717d3f4a54e02"
+ dependencies:
+ hoek "4.x.x"
+
bower-config@0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/bower-config/-/bower-config-0.6.1.tgz#7093155688bef44079bf4cb32d189312c87ded60"
@@ -507,6 +552,13 @@ brace-expansion@^1.0.0:
balanced-match "^0.4.1"
concat-map "0.0.1"
+brace-expansion@^1.1.7:
+ version "1.1.8"
+ resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292"
+ dependencies:
+ balanced-match "^1.0.0"
+ concat-map "0.0.1"
+
braces@^1.8.2:
version "1.8.5"
resolved "https://registry.yarnpkg.com/braces/-/braces-1.8.5.tgz#ba77962e12dff969d6b76711e914b737857bf6a7"
@@ -558,6 +610,17 @@ broccoli-caching-writer@^2.0.0, broccoli-caching-writer@^2.0.4, broccoli-caching
rsvp "^3.0.17"
walk-sync "^0.2.5"
+broccoli-caching-writer@^3.0.3:
+ version "3.0.3"
+ resolved "https://registry.yarnpkg.com/broccoli-caching-writer/-/broccoli-caching-writer-3.0.3.tgz#0bd2c96a9738d6a6ab590f07ba35c5157d7db476"
+ dependencies:
+ broccoli-kitchen-sink-helpers "^0.3.1"
+ broccoli-plugin "^1.2.1"
+ debug "^2.1.1"
+ rimraf "^2.2.8"
+ rsvp "^3.0.17"
+ walk-sync "^0.3.0"
+
broccoli-clean-css@0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/broccoli-clean-css/-/broccoli-clean-css-0.2.0.tgz#15f1c265a6986585a972bfb070bf52e9c054c861"
@@ -716,6 +779,17 @@ broccoli-sane-watcher@^1.1.1:
rsvp "^3.0.18"
sane "^1.1.1"
+broccoli-sass-source-maps@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/broccoli-sass-source-maps/-/broccoli-sass-source-maps-2.0.0.tgz#7f25f9f4b296918cec6e00672c63e75abce33d45"
+ dependencies:
+ broccoli-caching-writer "^3.0.3"
+ include-path-searcher "^0.1.0"
+ mkdirp "^0.3.5"
+ node-sass "^4.1.0"
+ object-assign "^2.0.0"
+ rsvp "^3.0.6"
+
broccoli-slow-trees@^1.0.0, broccoli-slow-trees@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/broccoli-slow-trees/-/broccoli-slow-trees-1.1.0.tgz#426c5724e008107e4573f73e8a9ca702916b78f7"
@@ -860,10 +934,25 @@ callsite@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/callsite/-/callsite-1.0.0.tgz#280398e5d664bd74038b6f0905153e6e8af1bc20"
+camelcase-keys@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/camelcase-keys/-/camelcase-keys-2.1.0.tgz#308beeaffdf28119051efa1d932213c91b8f92e7"
+ dependencies:
+ camelcase "^2.0.0"
+ map-obj "^1.0.0"
+
camelcase@^1.0.2, camelcase@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-1.2.1.tgz#9bb5304d2e0b56698b2c758b08a3eaa9daa58a39"
+camelcase@^2.0.0:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-2.1.1.tgz#7c1d16d679a1bbe59ca02cacecfb011e201f5a1f"
+
+camelcase@^3.0.0:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
+
can-symlink@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/can-symlink/-/can-symlink-1.0.0.tgz#97b607d8a84bb6c6e228b902d864ecb594b9d219"
@@ -881,6 +970,10 @@ caseless@~0.11.0:
version "0.11.0"
resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.11.0.tgz#715b96ea9841593cc33067923f5ec60ebda4f7d7"
+caseless@~0.12.0:
+ version "0.12.0"
+ resolved "https://registry.yarnpkg.com/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc"
+
center-align@^0.1.1:
version "0.1.3"
resolved "https://registry.yarnpkg.com/center-align/-/center-align-0.1.3.tgz#aa0d32629b6ee972200411cbd4461c907bc2b7ad"
@@ -984,6 +1077,14 @@ cliui@^2.1.0:
right-align "^0.1.1"
wordwrap "0.0.2"
+cliui@^3.2.0:
+ version "3.2.0"
+ resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d"
+ dependencies:
+ string-width "^1.0.1"
+ strip-ansi "^3.0.1"
+ wrap-ansi "^2.0.0"
+
clone@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/clone/-/clone-0.2.0.tgz#c6126a90ad4f72dbf5acdb243cc37724fe93fc1f"
@@ -999,6 +1100,14 @@ cmd-shim@~2.0.1:
graceful-fs "^4.1.2"
mkdirp "~0.5.0"
+co@^4.6.0:
+ version "4.6.0"
+ resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184"
+
+code-point-at@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/code-point-at/-/code-point-at-1.1.0.tgz#0d070b4d043a5bea33a2f1a40e2edb3d9a4ccf77"
+
colors@1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/colors/-/colors-1.0.3.tgz#0433f44d809680fdeb60ed260f1b0c262e82a40b"
@@ -1124,6 +1233,10 @@ console-browserify@1.1.x:
dependencies:
date-now "^0.1.4"
+console-control-strings@^1.0.0, console-control-strings@~1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/console-control-strings/-/console-control-strings-1.1.0.tgz#3d7cf4464db6446ea644bf4b39507f9851008e8e"
+
consolidate@^0.13.1:
version "0.13.1"
resolved "https://registry.yarnpkg.com/consolidate/-/consolidate-0.13.1.tgz#9e9503568eb4850889da6ed87a852c8dd2d13f64"
@@ -1183,16 +1296,35 @@ cross-spawn-async@^2.0.0:
lru-cache "^4.0.0"
which "^1.2.8"
+cross-spawn@^3.0.0:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-3.0.1.tgz#1256037ecb9f0c5f79e3d6ef135e30770184b982"
+ dependencies:
+ lru-cache "^4.0.1"
+ which "^1.2.9"
+
cryptiles@2.x.x:
version "2.0.5"
resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8"
dependencies:
boom "2.x.x"
+cryptiles@3.x.x:
+ version "3.1.2"
+ resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-3.1.2.tgz#a89fbb220f5ce25ec56e8c4aa8a4fd7b5b0d29fe"
+ dependencies:
+ boom "5.x.x"
+
ctype@0.5.3:
version "0.5.3"
resolved "https://registry.yarnpkg.com/ctype/-/ctype-0.5.3.tgz#82c18c2461f74114ef16c135224ad0b9144ca12f"
+currently-unhandled@^0.4.1:
+ version "0.4.1"
+ resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea"
+ dependencies:
+ array-find-index "^1.0.1"
+
d@1:
version "1.0.0"
resolved "https://registry.yarnpkg.com/d/-/d-1.0.0.tgz#754bb5bfe55451da69a58b94d45f4c5b0462d58f"
@@ -1271,7 +1403,7 @@ debuglog@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/debuglog/-/debuglog-1.0.1.tgz#aa24ffb9ac3df9a2351837cfb2d279360cd78492"
-decamelize@^1.0.0:
+decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
version "1.2.0"
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
@@ -1599,6 +1731,15 @@ ember-cli-release@0.2.8:
semver "^4.3.1"
silent-error "^1.0.0"
+ember-cli-sass@=7.0.0:
+ version "7.0.0"
+ resolved "https://registry.yarnpkg.com/ember-cli-sass/-/ember-cli-sass-7.0.0.tgz#24bdf6e6c13610c57cbddb1f077715588aedcf13"
+ dependencies:
+ broccoli-funnel "^1.0.0"
+ broccoli-merge-trees "^1.1.0"
+ broccoli-sass-source-maps "^2.0.0"
+ ember-cli-version-checker "^1.0.2"
+
ember-cli-sri@1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ember-cli-sri/-/ember-cli-sri-1.2.1.tgz#105b1f8bfb88fff8817caa14d0776ecb06f857ee"
@@ -1839,6 +1980,12 @@ errno@^0.1.1:
dependencies:
prr "~0.0.0"
+error-ex@^1.2.0:
+ version "1.3.1"
+ resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.1.tgz#f855a86ce61adc4e8621c3cda21e7a7612c3a8dc"
+ dependencies:
+ is-arrayish "^0.2.1"
+
es5-ext@^0.10.14, es5-ext@^0.10.9, es5-ext@~0.10.11, es5-ext@~0.10.14, es5-ext@~0.10.2, es5-ext@~0.10.5, es5-ext@~0.10.6:
version "0.10.15"
resolved "https://registry.yarnpkg.com/es5-ext/-/es5-ext-0.10.15.tgz#c330a5934c1ee21284a7c081a86e5fd937c91ea6"
@@ -1999,6 +2146,10 @@ extend@~3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4"
+extend@~3.0.1:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.1.tgz#a755ea7bc1adfcc5a31ce7e762dbaadc5e636444"
+
extglob@^0.3.1:
version "0.3.2"
resolved "https://registry.yarnpkg.com/extglob/-/extglob-0.3.2.tgz#2e18ff3d2f49ab2765cec9023f011daa8d8349a1"
@@ -2018,6 +2169,14 @@ extsprintf@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.0.2.tgz#e1080e0658e300b06294990cc70e1502235fd550"
+fast-deep-equal@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz#96256a3bc975595eb36d82e9929d060d893439ff"
+
+fast-json-stable-stringify@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz#d5142c0caee6b1189f87d3a76111064f86c8bbf2"
+
fast-ordered-set@^1.0.0, fast-ordered-set@^1.0.2:
version "1.0.3"
resolved "https://registry.yarnpkg.com/fast-ordered-set/-/fast-ordered-set-1.0.3.tgz#3fbb36634f7be79e4f7edbdb4a357dee25d184eb"
@@ -2098,6 +2257,13 @@ finalhandler@~1.0.0:
statuses "~1.3.1"
unpipe "~1.0.0"
+find-up@^1.0.0:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/find-up/-/find-up-1.1.2.tgz#6b2e9822b1a2ce0a60ab64d610eccad53cb24d0f"
+ dependencies:
+ path-exists "^2.0.0"
+ pinkie-promise "^2.0.0"
+
findup-sync@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/findup-sync/-/findup-sync-0.2.1.tgz#e0a90a450075c49466ee513732057514b81e878c"
@@ -2148,6 +2314,14 @@ form-data@~1.0.0-rc3, form-data@~1.0.0-rc4:
combined-stream "^1.0.5"
mime-types "^2.1.11"
+form-data@~2.3.1:
+ version "2.3.1"
+ resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.1.tgz#6fb94fbd71885306d73d15cc497fe4cc4ecd44bf"
+ dependencies:
+ asynckit "^0.4.0"
+ combined-stream "^1.0.5"
+ mime-types "^2.1.12"
+
forwarded@~0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/forwarded/-/forwarded-0.1.0.tgz#19ef9874c4ae1c297bcf078fde63a09b66a84363"
@@ -2284,6 +2458,25 @@ gauge@~1.2.0, gauge@~1.2.5:
lodash.padend "^4.1.0"
lodash.padstart "^4.1.0"
+gauge@~2.7.3:
+ version "2.7.4"
+ resolved "https://registry.yarnpkg.com/gauge/-/gauge-2.7.4.tgz#2c03405c7538c39d7eb37b317022e325fb018bf7"
+ dependencies:
+ aproba "^1.0.3"
+ console-control-strings "^1.0.0"
+ has-unicode "^2.0.0"
+ object-assign "^4.1.0"
+ signal-exit "^3.0.0"
+ string-width "^1.0.1"
+ strip-ansi "^3.0.1"
+ wide-align "^1.1.0"
+
+gaze@^1.0.0:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/gaze/-/gaze-1.1.2.tgz#847224677adb8870d679257ed3388fdb61e40105"
+ dependencies:
+ globule "^1.0.0"
+
generate-function@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/generate-function/-/generate-function-2.0.0.tgz#6858fe7c0969b7d4e9093337647ac79f60dfbe74"
@@ -2294,6 +2487,10 @@ generate-object-property@^1.1.0:
dependencies:
is-property "^1.0.0"
+get-caller-file@^1.0.1:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-1.0.2.tgz#f702e63127e7e231c160a80c1554acb70d5047e5"
+
get-stdin@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-4.0.1.tgz#b968c6b0a04384324902e8bf1a5df32579a450fe"
@@ -2380,6 +2577,17 @@ glob@^6.0.1:
once "^1.3.0"
path-is-absolute "^1.0.0"
+glob@^7.0.0, glob@^7.0.3, glob@~7.1.1:
+ version "7.1.2"
+ resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
+ dependencies:
+ fs.realpath "^1.0.0"
+ inflight "^1.0.4"
+ inherits "2"
+ minimatch "^3.0.4"
+ once "^1.3.0"
+ path-is-absolute "^1.0.0"
+
glob@^7.0.5, glob@^7.1.1:
version "7.1.1"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.1.tgz#805211df04faaf1c63a3600306cdf5ade50b2ec8"
@@ -2395,6 +2603,14 @@ globals@^6.4.0:
version "6.4.1"
resolved "https://registry.yarnpkg.com/globals/-/globals-6.4.1.tgz#8498032b3b6d1cc81eebc5f79690d8fe29fabf4f"
+globule@^1.0.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/globule/-/globule-1.2.0.tgz#1dc49c6822dd9e8a2fa00ba2a295006e8664bd09"
+ dependencies:
+ glob "~7.1.1"
+ lodash "~4.17.4"
+ minimatch "~3.0.2"
+
graceful-fs@^3.0.5:
version "3.0.11"
resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-3.0.11.tgz#7613c778a1afea62f25c630a086d7f3acbbdd818"
@@ -2426,6 +2642,10 @@ handlebars@^3.0.1:
optionalDependencies:
uglify-js "~2.3"
+har-schema@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/har-schema/-/har-schema-2.0.0.tgz#a94c2224ebcac04782a0d9035521f24735b7ec92"
+
har-validator@~2.0.2, har-validator@~2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-2.0.6.tgz#cdcbc08188265ad119b6a5a7c8ab70eecfb5d27d"
@@ -2435,6 +2655,13 @@ har-validator@~2.0.2, har-validator@~2.0.6:
is-my-json-valid "^2.12.4"
pinkie-promise "^2.0.0"
+har-validator@~5.0.3:
+ version "5.0.3"
+ resolved "https://registry.yarnpkg.com/har-validator/-/har-validator-5.0.3.tgz#ba402c266194f15956ef15e0fcf242993f6a7dfd"
+ dependencies:
+ ajv "^5.1.0"
+ har-schema "^2.0.0"
+
has-ansi@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/has-ansi/-/has-ansi-0.1.0.tgz#84f265aae8c0e6a88a12d7022894b7568894c62e"
@@ -2496,6 +2723,15 @@ hawk@~3.1.0, hawk@~3.1.3:
hoek "2.x.x"
sntp "1.x.x"
+hawk@~6.0.2:
+ version "6.0.2"
+ resolved "https://registry.yarnpkg.com/hawk/-/hawk-6.0.2.tgz#af4d914eb065f9b5ce4d9d11c1cb2126eecc3038"
+ dependencies:
+ boom "4.x.x"
+ cryptiles "3.x.x"
+ hoek "4.x.x"
+ sntp "2.x.x"
+
heimdalljs-logger@^0.1.7:
version "0.1.9"
resolved "https://registry.yarnpkg.com/heimdalljs-logger/-/heimdalljs-logger-0.1.9.tgz#d76ada4e45b7bb6f786fc9c010a68eb2e2faf176"
@@ -2513,6 +2749,10 @@ hoek@2.x.x:
version "2.16.3"
resolved "https://registry.yarnpkg.com/hoek/-/hoek-2.16.3.tgz#20bb7403d3cea398e91dc4710a8ff1b8274a25ed"
+hoek@4.x.x:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.0.tgz#72d9d0754f7fe25ca2d01ad8f8f9a9449a89526d"
+
home-or-tmp@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/home-or-tmp/-/home-or-tmp-1.0.0.tgz#4b9f1e40800c3e50c6c27f781676afcce71f3985"
@@ -2573,6 +2813,14 @@ http-signature@~1.1.0:
jsprim "^1.2.2"
sshpk "^1.7.0"
+http-signature@~1.2.0:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1"
+ dependencies:
+ assert-plus "^1.0.0"
+ jsprim "^1.2.2"
+ sshpk "^1.7.0"
+
ic-ajax@~2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/ic-ajax/-/ic-ajax-2.0.2.tgz#50c8ab1c494320fac66830ce1ee3990ab827f596"
@@ -2597,10 +2845,20 @@ imurmurhash@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea"
+in-publish@^2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/in-publish/-/in-publish-2.0.0.tgz#e20ff5e3a2afc2690320b6dc552682a9c7fadf51"
+
include-path-searcher@^0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/include-path-searcher/-/include-path-searcher-0.1.0.tgz#c0cf2ddfa164fb2eae07bc7ca43a7f191cb4d7bd"
+indent-string@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/indent-string/-/indent-string-2.1.0.tgz#8e2d48348742121b4a8218b7a137e9a52049dc80"
+ dependencies:
+ repeating "^2.0.0"
+
indexof@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/indexof/-/indexof-0.0.1.tgz#82dc336d232b9062179d05ab3293a66059fd435d"
@@ -2616,7 +2874,7 @@ inflight@^1.0.4, inflight@~1.0.4:
once "^1.3.0"
wrappy "1"
-inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1:
+inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3:
version "2.0.3"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
@@ -2657,6 +2915,10 @@ ipaddr.js@1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.3.0.tgz#1e03a52fdad83a8bbb2b25cbf4998b4cffcd3dec"
+is-arrayish@^0.2.1:
+ version "0.2.1"
+ resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
+
is-buffer@^1.0.2:
version "1.1.5"
resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.5.tgz#1f3b26ef613b214b88cbca23cc6c01d87961eecc"
@@ -2691,6 +2953,12 @@ is-finite@^1.0.0:
dependencies:
number-is-nan "^1.0.0"
+is-fullwidth-code-point@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz#ef9e31386f031a7f0d643af82fde50c457ef00cb"
+ dependencies:
+ number-is-nan "^1.0.0"
+
is-git-url@0.2.0, is-git-url@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/is-git-url/-/is-git-url-0.2.0.tgz#b9ce0fb044821c88880213d602db03bdb255da1b"
@@ -2796,6 +3064,10 @@ jodid25519@^1.0.0:
dependencies:
jsbn "~0.1.0"
+js-base64@^2.1.8:
+ version "2.3.2"
+ resolved "https://registry.yarnpkg.com/js-base64/-/js-base64-2.3.2.tgz#a79a923666372b580f8e27f51845c6f7e8fbfbaf"
+
js-tokens@1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-1.0.1.tgz#cc435a5c8b94ad15acb7983140fc80182c89aeae"
@@ -2834,6 +3106,10 @@ json-parse-helpfulerror@^1.0.2:
dependencies:
jju "^1.1.0"
+json-schema-traverse@^0.3.0:
+ version "0.3.1"
+ resolved "https://registry.yarnpkg.com/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz#349a6d44c53a51de89b40805c5d5e59b417d3340"
+
json-schema@0.2.3:
version "0.2.3"
resolved "https://registry.yarnpkg.com/json-schema/-/json-schema-0.2.3.tgz#b480c892e59a2f05954ce727bd3f2a4e882f9e13"
@@ -2945,6 +3221,16 @@ livereload-js@^2.2.0:
version "2.2.2"
resolved "https://registry.yarnpkg.com/livereload-js/-/livereload-js-2.2.2.tgz#6c87257e648ab475bc24ea257457edcc1f8d0bc2"
+load-json-file@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/load-json-file/-/load-json-file-1.1.0.tgz#956905708d58b4bab4c2261b04f59f31c99374c0"
+ dependencies:
+ graceful-fs "^4.1.2"
+ parse-json "^2.2.0"
+ pify "^2.0.0"
+ pinkie-promise "^2.0.0"
+ strip-bom "^2.0.0"
+
loader.js@4.2.3:
version "4.2.3"
resolved "https://registry.yarnpkg.com/loader.js/-/loader.js-4.2.3.tgz#845228877aa5317209e41f6c00d9bab36a6a4808"
@@ -3055,6 +3341,14 @@ lodash.assign@^3.0.0:
lodash._createassigner "^3.0.0"
lodash.keys "^3.0.0"
+lodash.assign@^4.2.0:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/lodash.assign/-/lodash.assign-4.2.0.tgz#0d99f3ccd7a6d261d19bdaeb9245005d285808e7"
+
+lodash.clonedeep@^4.3.2:
+ version "4.5.0"
+ resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef"
+
lodash.defaults@^3.1.2:
version "3.1.2"
resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-3.1.2.tgz#c7308b18dbf8bc9372d701a73493c61192bd2e2c"
@@ -3117,6 +3411,10 @@ lodash.merge@^4.5.1:
version "4.6.0"
resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.0.tgz#69884ba144ac33fe699737a6086deffadd0f89c5"
+lodash.mergewith@^4.6.0:
+ version "4.6.0"
+ resolved "https://registry.yarnpkg.com/lodash.mergewith/-/lodash.mergewith-4.6.0.tgz#150cf0a16791f5903b8891eab154609274bdea55"
+
lodash.pad@^4.1.0:
version "4.5.1"
resolved "https://registry.yarnpkg.com/lodash.pad/-/lodash.pad-4.5.1.tgz#4330949a833a7c8da22cc20f6a26c4d59debba70"
@@ -3164,7 +3462,7 @@ lodash@^3.10.0, lodash@^3.6.0, lodash@^3.9.3:
version "3.10.1"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-3.10.1.tgz#5bf45e8e49ba4189e17d482789dfd15bd140b7b6"
-lodash@^4.14.0, lodash@^4.5.1:
+lodash@^4.0.0, lodash@^4.14.0, lodash@^4.5.1, lodash@~4.17.4:
version "4.17.4"
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
@@ -3180,6 +3478,13 @@ longest@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/longest/-/longest-1.0.1.tgz#30a0b2da38f73770e8294a0d22e6625ed77d0097"
+loud-rejection@^1.0.0:
+ version "1.6.0"
+ resolved "https://registry.yarnpkg.com/loud-rejection/-/loud-rejection-1.6.0.tgz#5b46f80147edee578870f086d04821cf998e551f"
+ dependencies:
+ currently-unhandled "^0.4.1"
+ signal-exit "^3.0.0"
+
lru-cache@2, lru-cache@~2.7.0:
version "2.7.3"
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"
@@ -3191,6 +3496,13 @@ lru-cache@^4.0.0:
pseudomap "^1.0.1"
yallist "^2.0.0"
+lru-cache@^4.0.1:
+ version "4.1.1"
+ resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.1.tgz#622e32e82488b49279114a4f9ecf45e7cd6bba55"
+ dependencies:
+ pseudomap "^1.0.2"
+ yallist "^2.1.2"
+
lru-queue@0.1:
version "0.1.0"
resolved "https://registry.yarnpkg.com/lru-queue/-/lru-queue-0.1.0.tgz#2738bd9f0d3cf4f84490c5736c48699ac632cda3"
@@ -3207,6 +3519,10 @@ makeerror@1.0.x:
dependencies:
tmpl "1.0.x"
+map-obj@^1.0.0, map-obj@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/map-obj/-/map-obj-1.0.1.tgz#d933ceb9205d82bdcf4886f6742bdc2b4dea146d"
+
markdown-it-terminal@0.0.2:
version "0.0.2"
resolved "https://registry.yarnpkg.com/markdown-it-terminal/-/markdown-it-terminal-0.0.2.tgz#fff2c4a5df237800639b468a5fdd78e21917c4f2"
@@ -3263,6 +3579,21 @@ memoizee@~0.3.8:
next-tick "~0.2.2"
timers-ext "0.1"
+meow@^3.7.0:
+ version "3.7.0"
+ resolved "https://registry.yarnpkg.com/meow/-/meow-3.7.0.tgz#72cb668b425228290abbfa856892587308a801fb"
+ dependencies:
+ camelcase-keys "^2.0.0"
+ decamelize "^1.1.2"
+ loud-rejection "^1.0.0"
+ map-obj "^1.0.1"
+ minimist "^1.1.3"
+ normalize-package-data "^2.3.4"
+ object-assign "^4.0.1"
+ read-pkg-up "^1.0.1"
+ redent "^1.0.0"
+ trim-newlines "^1.0.0"
+
merge-defaults@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/merge-defaults/-/merge-defaults-0.2.1.tgz#dd42248eb96bb6a51521724321c72ff9583dde80"
@@ -3303,12 +3634,22 @@ micromatch@^2.1.5:
version "1.27.0"
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.27.0.tgz#820f572296bbd20ec25ed55e5b5de869e5436eb1"
+mime-db@~1.30.0:
+ version "1.30.0"
+ resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.30.0.tgz#74c643da2dd9d6a45399963465b26d5ca7d71f01"
+
mime-types@^2.1.11, mime-types@~2.1.11, mime-types@~2.1.15, mime-types@~2.1.7:
version "2.1.15"
resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.15.tgz#a4ebf5064094569237b8cf70046776d09fc92aed"
dependencies:
mime-db "~1.27.0"
+mime-types@^2.1.12, mime-types@~2.1.17:
+ version "2.1.17"
+ resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.17.tgz#09d7a393f03e995a79f8af857b70a9e0ab16557a"
+ dependencies:
+ mime-db "~1.30.0"
+
mime@1.3.4, mime@^1.2.11:
version "1.3.4"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
@@ -3332,6 +3673,12 @@ minimatch@2.x, minimatch@^2.0.1, minimatch@^2.0.10, minimatch@^2.0.3, minimatch@
dependencies:
brace-expansion "^1.0.0"
+minimatch@^3.0.4:
+ version "3.0.4"
+ resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
+ dependencies:
+ brace-expansion "^1.1.7"
+
minimatch@~0.2.9:
version "0.2.14"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-0.2.14.tgz#c74e780574f63c6f9a090e90efbe6ef53a6a756a"
@@ -3343,7 +3690,7 @@ minimist@0.0.8, minimist@~0.0.1:
version "0.0.8"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
-minimist@^1.1.0, minimist@^1.1.1:
+minimist@^1.1.0, minimist@^1.1.1, minimist@^1.1.3:
version "1.2.0"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
@@ -3421,6 +3768,10 @@ mute-stream@0.0.4, mute-stream@~0.0.4:
version "0.0.4"
resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.4.tgz#a9219960a6d5d5d046597aee51252c6655f7177e"
+nan@^2.3.2:
+ version "2.7.0"
+ resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46"
+
natives@^1.1.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/natives/-/natives-1.1.0.tgz#e9ff841418a6b2ec7a495e939984f78f163e6e31"
@@ -3437,6 +3788,24 @@ next-tick@~0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-0.2.2.tgz#75da4a927ee5887e39065880065b7336413b310d"
+node-gyp@^3.3.1:
+ version "3.6.2"
+ resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.6.2.tgz#9bfbe54562286284838e750eac05295853fa1c60"
+ dependencies:
+ fstream "^1.0.0"
+ glob "^7.0.3"
+ graceful-fs "^4.1.2"
+ minimatch "^3.0.2"
+ mkdirp "^0.5.0"
+ nopt "2 || 3"
+ npmlog "0 || 1 || 2 || 3 || 4"
+ osenv "0"
+ request "2"
+ rimraf "2"
+ semver "~5.3.0"
+ tar "^2.0.0"
+ which "1"
+
node-gyp@~3.0.3:
version "3.0.3"
resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.0.3.tgz#9b004219f4fa9efbfd78c5fc674aa12e58fb8694"
@@ -3464,6 +3833,29 @@ node-modules-path@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/node-modules-path/-/node-modules-path-1.0.1.tgz#40096b08ce7ad0ea14680863af449c7c75a5d1c8"
+node-sass@^4.1.0:
+ version "4.6.0"
+ resolved "https://registry.yarnpkg.com/node-sass/-/node-sass-4.6.0.tgz#1a54f5f4502e3cde310a26d6346266fd667271d9"
+ dependencies:
+ async-foreach "^0.1.3"
+ chalk "^1.1.1"
+ cross-spawn "^3.0.0"
+ gaze "^1.0.0"
+ get-stdin "^4.0.1"
+ glob "^7.0.3"
+ in-publish "^2.0.0"
+ lodash.assign "^4.2.0"
+ lodash.clonedeep "^4.3.2"
+ lodash.mergewith "^4.6.0"
+ meow "^3.7.0"
+ mkdirp "^0.5.1"
+ nan "^2.3.2"
+ node-gyp "^3.3.1"
+ npmlog "^4.0.0"
+ request "^2.79.0"
+ sass-graph "^2.2.4"
+ stdout-stream "^1.4.0"
+
node-uuid@^1.4.3, node-uuid@~1.4.3, node-uuid@~1.4.7:
version "1.4.8"
resolved "https://registry.yarnpkg.com/node-uuid/-/node-uuid-1.4.8.tgz#b040eb0923968afabf8d32fb1f17f1167fdab907"
@@ -3487,6 +3879,15 @@ normalize-package-data@^2.0.0, "normalize-package-data@~1.0.1 || ^2.0.0", normal
semver "2 || 3 || 4 || 5"
validate-npm-package-license "^3.0.1"
+normalize-package-data@^2.3.2, normalize-package-data@^2.3.4:
+ version "2.4.0"
+ resolved "https://registry.yarnpkg.com/normalize-package-data/-/normalize-package-data-2.4.0.tgz#12f95a307d58352075a04907b84ac8be98ac012f"
+ dependencies:
+ hosted-git-info "^2.1.4"
+ is-builtin-module "^1.0.0"
+ semver "2 || 3 || 4 || 5"
+ validate-npm-package-license "^3.0.1"
+
normalize-path@^2.0.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/normalize-path/-/normalize-path-2.1.1.tgz#1ab28b556e198363a8c1a6f7e6fa20137fe6aed9"
@@ -3622,6 +4023,15 @@ npm@2.14.10:
are-we-there-yet "~1.0.0"
gauge "~1.2.0"
+"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.0:
+ version "4.1.2"
+ resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b"
+ dependencies:
+ are-we-there-yet "~1.1.2"
+ console-control-strings "~1.1.0"
+ gauge "~2.7.3"
+ set-blocking "~2.0.0"
+
"npmlog@0.1 || 1 || 2", npmlog@~2.0.0:
version "2.0.4"
resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-2.0.4.tgz#98b52530f2514ca90d09ec5b22c8846722375692"
@@ -3638,10 +4048,14 @@ numeral@^1.5.3:
version "1.5.6"
resolved "https://registry.yarnpkg.com/numeral/-/numeral-1.5.6.tgz#3831db968451b9cf6aff9bf95925f1ef8e37b33f"
-oauth-sign@~0.8.0, oauth-sign@~0.8.1:
+oauth-sign@~0.8.0, oauth-sign@~0.8.1, oauth-sign@~0.8.2:
version "0.8.2"
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
+object-assign@^2.0.0:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa"
+
object-assign@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-3.0.0.tgz#9bedd5ca0897949bca47e7ff408062d549f587f2"
@@ -3750,6 +4164,12 @@ parse-glob@^3.0.4:
is-extglob "^1.0.0"
is-glob "^2.0.0"
+parse-json@^2.2.0:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9"
+ dependencies:
+ error-ex "^1.2.0"
+
parsejson@0.0.1:
version "0.0.1"
resolved "https://registry.yarnpkg.com/parsejson/-/parsejson-0.0.1.tgz#9b10c6c0d825ab589e685153826de0a3ba278bcc"
@@ -3788,6 +4208,12 @@ path-exists@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-1.0.0.tgz#d5a8998eb71ef37a74c34eb0d9eba6e878eea081"
+path-exists@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-2.1.0.tgz#0feb6c64f0fc518d9a754dd5efb62c7022761f4b"
+ dependencies:
+ pinkie-promise "^2.0.0"
+
path-is-absolute@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
@@ -3808,10 +4234,22 @@ path-to-regexp@0.1.7:
version "0.1.7"
resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-0.1.7.tgz#df604178005f522f15eb4490e7247a1bfaa67f8c"
+path-type@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/path-type/-/path-type-1.1.0.tgz#59c44f7ee491da704da415da5a4070ba4f8fe441"
+ dependencies:
+ graceful-fs "^4.1.2"
+ pify "^2.0.0"
+ pinkie-promise "^2.0.0"
+
pend@~1.2.0:
version "1.2.0"
resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50"
+performance-now@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b"
+
phantomjs-prebuilt@2.1.13:
version "2.1.13"
resolved "https://registry.yarnpkg.com/phantomjs-prebuilt/-/phantomjs-prebuilt-2.1.13.tgz#66556ad9e965d893ca5a7dc9e763df7e8697f76d"
@@ -3826,6 +4264,10 @@ phantomjs-prebuilt@2.1.13:
request-progress "~2.0.1"
which "~1.2.10"
+pify@^2.0.0:
+ version "2.3.0"
+ resolved "https://registry.yarnpkg.com/pify/-/pify-2.3.0.tgz#ed141a6ac043a849ea588498e7dca8b15330e90c"
+
pinkie-promise@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/pinkie-promise/-/pinkie-promise-2.0.1.tgz#2135d6dfa7a358c069ac9b178776288228450ffa"
@@ -3906,7 +4348,7 @@ prr@~0.0.0:
version "0.0.0"
resolved "https://registry.yarnpkg.com/prr/-/prr-0.0.0.tgz#1a84b85908325501411853d0081ee3fa86e2926a"
-pseudomap@^1.0.1:
+pseudomap@^1.0.1, pseudomap@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
@@ -3934,6 +4376,10 @@ qs@~6.2.0:
version "6.2.3"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.2.3.tgz#1cfcb25c10a9b2b483053ff39f5dfc9233908cfe"
+qs@~6.5.1:
+ version "6.5.1"
+ resolved "https://registry.yarnpkg.com/qs/-/qs-6.5.1.tgz#349cdf6eef89ec45c12d7d5eb3fc0c870343a6d8"
+
quick-temp@0.1.3, quick-temp@^0.1.0, quick-temp@^0.1.2, quick-temp@^0.1.3:
version "0.1.3"
resolved "https://registry.yarnpkg.com/quick-temp/-/quick-temp-0.1.3.tgz#48c1886114691d74b96003de9322a7c992022af4"
@@ -3996,6 +4442,21 @@ read-installed@~4.0.3:
optionalDependencies:
graceful-fs "^4.1.2"
+read-pkg-up@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/read-pkg-up/-/read-pkg-up-1.0.1.tgz#9d63c13276c065918d57f002a57f40a1b643fb02"
+ dependencies:
+ find-up "^1.0.0"
+ read-pkg "^1.0.0"
+
+read-pkg@^1.0.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/read-pkg/-/read-pkg-1.1.0.tgz#f5ffaa5ecd29cb31c0474bca7d756b6bb29e3f28"
+ dependencies:
+ load-json-file "^1.0.0"
+ normalize-package-data "^2.3.2"
+ path-type "^1.0.0"
+
read@1, read@~1.0.1, read@~1.0.7:
version "1.0.7"
resolved "https://registry.yarnpkg.com/read/-/read-1.0.7.tgz#b3da19bd052431a97671d44a42634adf710b40c4"
@@ -4023,6 +4484,18 @@ readable-stream@1.1, readable-stream@~1.1.13:
isarray "0.0.1"
string_decoder "~0.10.x"
+readable-stream@^2.0.1:
+ version "2.3.3"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.3.tgz#368f2512d79f9d46fdfc71349ae7878bbc1eb95c"
+ dependencies:
+ core-util-is "~1.0.0"
+ inherits "~2.0.3"
+ isarray "~1.0.0"
+ process-nextick-args "~1.0.6"
+ safe-buffer "~5.1.1"
+ string_decoder "~1.0.3"
+ util-deprecate "~1.0.1"
+
readable-stream@~2.0.0, readable-stream@~2.0.5:
version "2.0.6"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.0.6.tgz#8f90341e68a53ccc928788dacfcd11b36eb9b78e"
@@ -4075,6 +4548,13 @@ recast@^0.11.17, recast@^0.11.3:
private "~0.1.5"
source-map "~0.5.0"
+redent@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde"
+ dependencies:
+ indent-string "^2.1.0"
+ strip-indent "^1.0.1"
+
redeyed@~0.5.0:
version "0.5.0"
resolved "https://registry.yarnpkg.com/redeyed/-/redeyed-0.5.0.tgz#7ab000e60ee3875ac115d29edb32c1403c6c25d1"
@@ -4141,6 +4621,12 @@ repeating@^1.1.0, repeating@^1.1.2:
dependencies:
is-finite "^1.0.0"
+repeating@^2.0.0:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/repeating/-/repeating-2.0.1.tgz#5214c53a926d3552707527fbab415dbc08d06dda"
+ dependencies:
+ is-finite "^1.0.0"
+
request-progress@~2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/request-progress/-/request-progress-2.0.1.tgz#5d36bb57961c673aa5b788dbc8141fdf23b44e08"
@@ -4173,6 +4659,33 @@ request@2, request@^2.27.0, request@^2.47.0, request@^2.72.0, request@~2.74.0:
tough-cookie "~2.3.0"
tunnel-agent "~0.4.1"
+request@^2.79.0:
+ version "2.83.0"
+ resolved "https://registry.yarnpkg.com/request/-/request-2.83.0.tgz#ca0b65da02ed62935887808e6f510381034e3356"
+ dependencies:
+ aws-sign2 "~0.7.0"
+ aws4 "^1.6.0"
+ caseless "~0.12.0"
+ combined-stream "~1.0.5"
+ extend "~3.0.1"
+ forever-agent "~0.6.1"
+ form-data "~2.3.1"
+ har-validator "~5.0.3"
+ hawk "~6.0.2"
+ http-signature "~1.2.0"
+ is-typedarray "~1.0.0"
+ isstream "~0.1.2"
+ json-stringify-safe "~5.0.1"
+ mime-types "~2.1.17"
+ oauth-sign "~0.8.2"
+ performance-now "^2.1.0"
+ qs "~6.5.1"
+ safe-buffer "^5.1.1"
+ stringstream "~0.0.5"
+ tough-cookie "~2.3.3"
+ tunnel-agent "^0.6.0"
+ uuid "^3.1.0"
+
request@~2.65.0:
version "2.65.0"
resolved "https://registry.yarnpkg.com/request/-/request-2.65.0.tgz#cc1a3bc72b96254734fc34296da322f9486ddeba"
@@ -4197,6 +4710,14 @@ request@~2.65.0:
tough-cookie "~2.2.0"
tunnel-agent "~0.4.1"
+require-directory@^2.1.1:
+ version "2.1.1"
+ resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"
+
+require-main-filename@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1"
+
requires-port@1.x.x:
version "1.0.0"
resolved "https://registry.yarnpkg.com/requires-port/-/requires-port-1.0.0.tgz#925d2601d39ac485e091cf0da5c6e694dc3dcaff"
@@ -4245,6 +4766,10 @@ rsvp@~3.2.1:
version "3.2.1"
resolved "https://registry.yarnpkg.com/rsvp/-/rsvp-3.2.1.tgz#07cb4a5df25add9e826ebc67dcc9fd89db27d84a"
+safe-buffer@^5.0.1, safe-buffer@^5.1.1, safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+ version "5.1.1"
+ resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.1.tgz#893312af69b2123def71f57889001671eeb2c853"
+
sane@^1.1.1:
version "1.6.0"
resolved "https://registry.yarnpkg.com/sane/-/sane-1.6.0.tgz#9610c452307a135d29c1fdfe2547034180c46775"
@@ -4263,11 +4788,27 @@ sanitize-filename@^1.5.3:
dependencies:
truncate-utf8-bytes "^1.0.0"
+sass-graph@^2.2.4:
+ version "2.2.4"
+ resolved "https://registry.yarnpkg.com/sass-graph/-/sass-graph-2.2.4.tgz#13fbd63cd1caf0908b9fd93476ad43a51d1e0b49"
+ dependencies:
+ glob "^7.0.0"
+ lodash "^4.0.0"
+ scss-tokenizer "^0.2.3"
+ yargs "^7.0.0"
+
+scss-tokenizer@^0.2.3:
+ version "0.2.3"
+ resolved "https://registry.yarnpkg.com/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz#8eb06db9a9723333824d3f5530641149847ce5d1"
+ dependencies:
+ js-base64 "^2.1.8"
+ source-map "^0.4.2"
+
select2@4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/select2/-/select2-4.0.0.tgz#81b461456c77148e7e422eb63bd71ca0b254fce7"
-"semver@2 >=2.2.1 || 3.x || 4 || 5", "semver@2 || 3 || 4 || 5", "semver@2.x || 3.x || 4 || 5", "semver@4 || 5", "semver@^2.3.0 || 3.x || 4 || 5", semver@^5.1.0, semver@^5.3.0:
+"semver@2 >=2.2.1 || 3.x || 4 || 5", "semver@2 || 3 || 4 || 5", "semver@2.x || 3.x || 4 || 5", "semver@4 || 5", "semver@^2.3.0 || 3.x || 4 || 5", semver@^5.1.0, semver@^5.3.0, semver@~5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
@@ -4306,6 +4847,10 @@ serve-static@1.12.1:
parseurl "~1.3.1"
send "0.15.1"
+set-blocking@^2.0.0, set-blocking@~2.0.0:
+ version "2.0.0"
+ resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7"
+
setprototypeof@1.0.3:
version "1.0.3"
resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.0.3.tgz#66567e37043eeb4f04d91bd658c0cbefb55b8e04"
@@ -4329,6 +4874,10 @@ sigmund@~1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/sigmund/-/sigmund-1.0.1.tgz#3ff21f198cad2175f9f3b781853fd94d0d19b590"
+signal-exit@^3.0.0:
+ version "3.0.2"
+ resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d"
+
silent-error@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/silent-error/-/silent-error-1.0.1.tgz#71b7d503d1c6f94882b51b56be879b113cb4822c"
@@ -4357,6 +4906,12 @@ sntp@1.x.x:
dependencies:
hoek "2.x.x"
+sntp@2.x.x:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/sntp/-/sntp-2.1.0.tgz#2c6cec14fedc2222739caf9b5c3d85d1cc5a2cc8"
+ dependencies:
+ hoek "4.x.x"
+
socket.io-adapter@0.3.1:
version "0.3.1"
resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-0.3.1.tgz#df81f970b4df6493902d93e519c7b72755c3a958"
@@ -4503,6 +5058,20 @@ statuses@1, "statuses@>= 1.3.1 < 2", statuses@~1.3.1:
version "1.3.1"
resolved "https://registry.yarnpkg.com/statuses/-/statuses-1.3.1.tgz#faf51b9eb74aaef3b3acf4ad5f61abf24cb7b93e"
+stdout-stream@^1.4.0:
+ version "1.4.0"
+ resolved "https://registry.yarnpkg.com/stdout-stream/-/stdout-stream-1.4.0.tgz#a2c7c8587e54d9427ea9edb3ac3f2cd522df378b"
+ dependencies:
+ readable-stream "^2.0.1"
+
+string-width@^1.0.1, string-width@^1.0.2:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
+ dependencies:
+ code-point-at "^1.0.0"
+ is-fullwidth-code-point "^1.0.0"
+ strip-ansi "^3.0.0"
+
string.prototype.endswith@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/string.prototype.endswith/-/string.prototype.endswith-0.2.0.tgz#a19c20dee51a98777e9a47e10f09be393b9bba75"
@@ -4517,6 +5086,12 @@ string_decoder@~1.0.0:
dependencies:
buffer-shims "~1.0.0"
+string_decoder@~1.0.3:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.0.3.tgz#0fc67d7c141825de94282dd536bec6b9bce860ab"
+ dependencies:
+ safe-buffer "~5.1.0"
+
stringmap@~0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/stringmap/-/stringmap-0.2.2.tgz#556c137b258f942b8776f5b2ef582aa069d7d1b1"
@@ -4525,7 +5100,7 @@ stringset@~0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/stringset/-/stringset-0.2.1.tgz#ef259c4e349344377fcd1c913dd2e848c9c042b5"
-stringstream@~0.0.4:
+stringstream@~0.0.4, stringstream@~0.0.5:
version "0.0.5"
resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
@@ -4541,7 +5116,7 @@ strip-ansi@^2.0.1:
dependencies:
ansi-regex "^1.0.0"
-strip-ansi@^3.0.0:
+strip-ansi@^3.0.0, strip-ansi@^3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-3.0.1.tgz#6a385fb8853d952d5ff05d0e8aaf94278dc63dcf"
dependencies:
@@ -4557,6 +5132,12 @@ strip-bom@^2.0.0:
dependencies:
is-utf8 "^0.2.0"
+strip-indent@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/strip-indent/-/strip-indent-1.0.1.tgz#0c7962a6adefa7bbd4ac366460a638552ae1a0a2"
+ dependencies:
+ get-stdin "^4.0.1"
+
strip-json-comments@1.0.x:
version "1.0.4"
resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-1.0.4.tgz#1e15fbcac97d3ee99bf2d73b4c656b082bbafb91"
@@ -4595,7 +5176,7 @@ tar@^1.0.0:
fstream "^1.0.2"
inherits "2"
-tar@~2.2.1:
+tar@^2.0.0, tar@~2.2.1:
version "2.2.1"
resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.1.tgz#8e4d2a256c0e2185c6b18ad694aec968b83cb1d1"
dependencies:
@@ -4698,6 +5279,16 @@ tough-cookie@~2.3.0:
dependencies:
punycode "^1.4.1"
+tough-cookie@~2.3.3:
+ version "2.3.3"
+ resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.3.3.tgz#0b618a5565b6dea90bf3425d04d55edc475a7561"
+ dependencies:
+ punycode "^1.4.1"
+
+trim-newlines@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-1.0.0.tgz#5887966bb582a4503a41eb524f7d35011815a613"
+
trim-right@^1.0.0:
version "1.0.1"
resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003"
@@ -4716,6 +5307,12 @@ tryor@~0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/tryor/-/tryor-0.1.2.tgz#8145e4ca7caff40acde3ccf946e8b8bb75b4172b"
+tunnel-agent@^0.6.0:
+ version "0.6.0"
+ resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd"
+ dependencies:
+ safe-buffer "^5.0.1"
+
tunnel-agent@~0.4.1:
version "0.4.3"
resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.4.3.tgz#6373db76909fe570e08d73583365ed828a74eeeb"
@@ -4812,6 +5409,10 @@ uuid@^2.0.1:
version "2.0.3"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-2.0.3.tgz#67e2e863797215530dff318e5bf9dcebfd47b21a"
+uuid@^3.1.0:
+ version "3.1.0"
+ resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04"
+
validate-npm-package-license@^3.0.1, validate-npm-package-license@~3.0.1:
version "3.0.1"
resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc"
@@ -4885,12 +5486,28 @@ websocket-extensions@>=0.1.1:
version "0.1.1"
resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.1.tgz#76899499c184b6ef754377c2dbb0cd6cb55d29e7"
+which-module@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/which-module/-/which-module-1.0.0.tgz#bba63ca861948994ff307736089e3b96026c2a4f"
+
which@1, which@^1.2.8, which@~1.2.0, which@~1.2.10:
version "1.2.14"
resolved "https://registry.yarnpkg.com/which/-/which-1.2.14.tgz#9a87c4378f03e827cecaf1acdf56c736c01c14e5"
dependencies:
isexe "^2.0.0"
+which@^1.2.9:
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/which/-/which-1.3.0.tgz#ff04bdfc010ee547d780bec38e1ac1c2777d253a"
+ dependencies:
+ isexe "^2.0.0"
+
+wide-align@^1.1.0:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/wide-align/-/wide-align-1.1.2.tgz#571e0f1b0604636ebc0dfc21b0339bbe31341710"
+ dependencies:
+ string-width "^1.0.2"
+
window-size@0.1.0:
version "0.1.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"
@@ -4907,6 +5524,13 @@ wordwrap@~0.0.2:
version "0.0.3"
resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107"
+wrap-ansi@^2.0.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"
+ dependencies:
+ string-width "^1.0.1"
+ strip-ansi "^3.0.1"
+
wrappy@1, wrappy@~1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
@@ -4952,11 +5576,11 @@ xtend@^4.0.0:
version "4.0.1"
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.1.tgz#a5c6d532be656e23db820efb943a1f04998d63af"
-y18n@^3.2.0:
+y18n@^3.2.0, y18n@^3.2.1:
version "3.2.1"
resolved "https://registry.yarnpkg.com/y18n/-/y18n-3.2.1.tgz#6d15fba884c08679c0d77e88e7759e811e07fa41"
-yallist@^2.0.0:
+yallist@^2.0.0, yallist@^2.1.2:
version "2.1.2"
resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52"
@@ -4968,6 +5592,30 @@ yam@0.0.18:
fs-extra "^0.16.3"
lodash.merge "^3.0.2"
+yargs-parser@^5.0.0:
+ version "5.0.0"
+ resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-5.0.0.tgz#275ecf0d7ffe05c77e64e7c86e4cd94bf0e1228a"
+ dependencies:
+ camelcase "^3.0.0"
+
+yargs@^7.0.0:
+ version "7.1.0"
+ resolved "https://registry.yarnpkg.com/yargs/-/yargs-7.1.0.tgz#6ba318eb16961727f5d284f8ea003e8d6154d0c8"
+ dependencies:
+ camelcase "^3.0.0"
+ cliui "^3.2.0"
+ decamelize "^1.1.1"
+ get-caller-file "^1.0.1"
+ os-locale "^1.4.0"
+ read-pkg-up "^1.0.1"
+ require-directory "^2.1.1"
+ require-main-filename "^1.0.1"
+ set-blocking "^2.0.0"
+ string-width "^1.0.2"
+ which-module "^1.0.0"
+ y18n "^3.2.1"
+ yargs-parser "^5.0.0"
+
yargs@~3.10.0:
version "3.10.0"
resolved "https://registry.yarnpkg.com/yargs/-/yargs-3.10.0.tgz#f7ee7bd857dd7c1d2d38c0e74efbd681d1431fd1"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/19] hadoop git commit: YARN-7486. Race condition in service AM
that can cause NPE. Contributed by Jian He
Posted by as...@apache.org.
YARN-7486. Race condition in service AM that can cause NPE. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4d5d202
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4d5d202
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4d5d202
Branch: refs/heads/YARN-6592
Commit: f4d5d20286eb05449f6fd7cd6ff0554228205fe2
Parents: 462e25a
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Nov 15 10:20:46 2017 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Thu Nov 16 07:58:06 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceScheduler.java | 50 ++++-----
.../yarn/service/component/Component.java | 58 ++++------
.../yarn/service/component/ComponentEvent.java | 11 ++
.../component/instance/ComponentInstance.java | 83 +++++++-------
.../containerlaunch/ContainerLaunchService.java | 2 +-
.../provider/AbstractProviderService.java | 5 +-
.../yarn/service/provider/ProviderService.java | 5 +-
.../yarn/service/provider/ProviderUtils.java | 5 +-
.../ServiceTimelinePublisher.java | 5 +-
.../hadoop/yarn/service/MockServiceAM.java | 66 ++++++++---
.../hadoop/yarn/service/ServiceTestUtils.java | 5 +-
.../hadoop/yarn/service/TestServiceAM.java | 109 +++++++++++++++++++
.../service/monitor/TestServiceMonitor.java | 12 ++
13 files changed, 290 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index a7b7e22..6bc5673 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -132,7 +132,6 @@ public class ServiceScheduler extends CompositeService {
private AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient;
private NMClientAsync nmClient;
private AsyncDispatcher dispatcher;
- AsyncDispatcher compInstanceDispatcher;
private YarnRegistryViewForProviders yarnRegistryOperations;
private ServiceContext context;
private ContainerLaunchService containerLaunchService;
@@ -152,7 +151,7 @@ public class ServiceScheduler extends CompositeService {
yarnRegistryOperations =
createYarnRegistryOperations(context, registryClient);
- // register metrics
+ // register metrics,
serviceMetrics = ServiceMetrics
.register(app.getName(), "Metrics for service");
serviceMetrics.tag("type", "Metrics type [component or service]", "service");
@@ -167,14 +166,11 @@ public class ServiceScheduler extends CompositeService {
dispatcher = new AsyncDispatcher("Component dispatcher");
dispatcher.register(ComponentEventType.class,
new ComponentEventHandler());
+ dispatcher.register(ComponentInstanceEventType.class,
+ new ComponentInstanceEventHandler());
dispatcher.setDrainEventsOnStop();
addIfService(dispatcher);
- compInstanceDispatcher =
- new AsyncDispatcher("CompInstance dispatcher");
- compInstanceDispatcher.register(ComponentInstanceEventType.class,
- new ComponentInstanceEventHandler());
- addIfService(compInstanceDispatcher);
containerLaunchService = new ContainerLaunchService(context.fs);
addService(containerLaunchService);
@@ -277,10 +273,10 @@ public class ServiceScheduler extends CompositeService {
}
private void recoverComponents(RegisterApplicationMasterResponse response) {
- List<Container> recoveredContainers = response
+ List<Container> containersFromPrevAttempt = response
.getContainersFromPreviousAttempts();
LOG.info("Received {} containers from previous attempt.",
- recoveredContainers.size());
+ containersFromPrevAttempt.size());
Map<String, ServiceRecord> existingRecords = new HashMap<>();
List<String> existingComps = null;
try {
@@ -302,9 +298,8 @@ public class ServiceScheduler extends CompositeService {
}
}
}
- for (Container container : recoveredContainers) {
- LOG.info("Handling container {} from previous attempt",
- container.getId());
+ for (Container container : containersFromPrevAttempt) {
+ LOG.info("Handling {} from previous attempt", container.getId());
ServiceRecord record = existingRecords.get(RegistryPathUtils
.encodeYarnID(container.getId().toString()));
if (record != null) {
@@ -487,16 +482,21 @@ public class ServiceScheduler extends CompositeService {
new ComponentEvent(comp.getName(), CONTAINER_ALLOCATED)
.setContainer(container);
dispatcher.getEventHandler().handle(event);
- Collection<AMRMClient.ContainerRequest> requests = amRMClient
- .getMatchingRequests(container.getAllocationRequestId());
- LOG.info("[COMPONENT {}]: {} outstanding container requests.",
- comp.getName(), requests.size());
- // remove the corresponding request
- if (requests.iterator().hasNext()) {
- LOG.info("[COMPONENT {}]: removing one container request.", comp
- .getName());
- AMRMClient.ContainerRequest request = requests.iterator().next();
- amRMClient.removeContainerRequest(request);
+ try {
+ Collection<AMRMClient.ContainerRequest> requests = amRMClient
+ .getMatchingRequests(container.getAllocationRequestId());
+ LOG.info("[COMPONENT {}]: remove {} outstanding container requests " +
+ "for allocateId " + container.getAllocationRequestId(),
+ comp.getName(), requests.size());
+ // remove the corresponding request
+ if (requests.iterator().hasNext()) {
+ AMRMClient.ContainerRequest request = requests.iterator().next();
+ amRMClient.removeContainerRequest(request);
+ }
+ } catch(Exception e) {
+ //TODO Due to YARN-7490, exception may be thrown, catch and ignore for
+ //now.
+ LOG.error("Exception when removing the matching requests. ", e);
}
}
}
@@ -569,7 +569,7 @@ public class ServiceScheduler extends CompositeService {
}
ComponentEvent event =
new ComponentEvent(instance.getCompName(), CONTAINER_STARTED)
- .setInstance(instance);
+ .setInstance(instance).setContainerId(containerId);
dispatcher.getEventHandler().handle(event);
}
@@ -649,10 +649,6 @@ public class ServiceScheduler extends CompositeService {
liveInstances.remove(containerId);
}
- public AsyncDispatcher getCompInstanceDispatcher() {
- return compInstanceDispatcher;
- }
-
public YarnRegistryViewForProviders getYarnRegistryOperations() {
return yarnRegistryOperations;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 7208f39..88f4763 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -82,7 +82,8 @@ public class Component implements EventHandler<ComponentEvent> {
private Map<String, ComponentInstance> compInstances =
new ConcurrentHashMap<>();
// component instances to be assigned with a container
- private List<ComponentInstance> pendingInstances = new LinkedList<>();
+ private List<ComponentInstance> pendingInstances =
+ Collections.synchronizedList(new LinkedList<>());
private ContainerFailureTracker failureTracker;
private Probe probe;
private final ReentrantReadWriteLock.ReadLock readLock;
@@ -94,7 +95,7 @@ public class Component implements EventHandler<ComponentEvent> {
private StateMachine<ComponentState, ComponentEventType, ComponentEvent>
stateMachine;
- private AsyncDispatcher compInstanceDispatcher;
+ private AsyncDispatcher dispatcher;
private static final StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>
stateMachineFactory =
new StateMachineFactory<Component, ComponentState, ComponentEventType, ComponentEvent>(
@@ -149,7 +150,7 @@ public class Component implements EventHandler<ComponentEvent> {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
this.stateMachine = stateMachineFactory.make(this);
- compInstanceDispatcher = scheduler.getCompInstanceDispatcher();
+ dispatcher = scheduler.getDispatcher();
failureTracker =
new ContainerFailureTracker(context, this);
probe = MonitorUtils.getProbe(componentSpec.getReadinessCheck());
@@ -256,30 +257,18 @@ public class Component implements EventHandler<ComponentEvent> {
component.releaseContainer(container);
return;
}
- if (instance.hasContainer()) {
- LOG.info(
- "[COMPONENT {}]: Instance {} already has container, release " +
- "surplus container {}",
- instance.getCompName(), instance.getCompInstanceId(), container
- .getId());
- component.releaseContainer(container);
- return;
- }
+
component.pendingInstances.remove(instance);
- LOG.info("[COMPONENT {}]: Recovered {} for component instance {} on " +
- "host {}, num pending component instances reduced to {} ",
- component.getName(), container.getId(), instance
- .getCompInstanceName(), container.getNodeId(), component
- .pendingInstances.size());
instance.setContainer(container);
ProviderUtils.initCompInstanceDir(component.getContext().fs, instance);
component.getScheduler().addLiveCompInstance(container.getId(), instance);
- LOG.info("[COMPONENT {}]: Marking {} as started for component " +
- "instance {}", component.getName(), event.getContainer().getId(),
- instance.getCompInstanceId());
- component.compInstanceDispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(instance.getContainerId(),
- START));
+ LOG.info("[COMPONENT {}]: Recovered {} for component instance {} on " +
+ "host {}, num pending component instances reduced to {} ",
+ component.getName(), container.getId(),
+ instance.getCompInstanceName(), container.getNodeId(),
+ component.pendingInstances.size());
+ component.dispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(container.getId(), START));
}
}
@@ -288,9 +277,8 @@ public class Component implements EventHandler<ComponentEvent> {
@Override public ComponentState transition(Component component,
ComponentEvent event) {
- component.compInstanceDispatcher.getEventHandler().handle(
- new ComponentInstanceEvent(event.getInstance().getContainerId(),
- START));
+ component.dispatcher.getEventHandler().handle(
+ new ComponentInstanceEvent(event.getContainerId(), START));
return checkIfStable(component);
}
}
@@ -313,14 +301,7 @@ public class Component implements EventHandler<ComponentEvent> {
@Override
public void transition(Component component, ComponentEvent event) {
component.updateMetrics(event.getStatus());
-
- // add back to pending list
- component.pendingInstances.add(event.getInstance());
- LOG.info(
- "[COMPONENT {}]: {} completed, num pending comp instances increased to {}.",
- component.getName(), event.getStatus().getContainerId(),
- component.pendingInstances.size());
- component.compInstanceDispatcher.getEventHandler().handle(
+ component.dispatcher.getEventHandler().handle(
new ComponentInstanceEvent(event.getStatus().getContainerId(),
STOP).setStatus(event.getStatus()));
component.componentSpec.setState(
@@ -328,8 +309,8 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
- public ServiceMetrics getCompMetrics () {
- return componentMetrics;
+ public void reInsertPendingInstance(ComponentInstance instance) {
+ pendingInstances.add(instance);
}
private void releaseContainer(Container container) {
@@ -581,4 +562,9 @@ public class Component implements EventHandler<ComponentEvent> {
public ServiceContext getContext() {
return context;
}
+
+ // Only for testing
+ public List<ComponentInstance> getPendingInstances() {
+ return pendingInstances;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
index d93dcf1..447b436 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.service.component;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.event.AbstractEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
@@ -30,6 +31,16 @@ public class ComponentEvent extends AbstractEvent<ComponentEventType> {
private Container container;
private ComponentInstance instance;
private ContainerStatus status;
+ private ContainerId containerId;
+
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ public ComponentEvent setContainerId(ContainerId containerId) {
+ this.containerId = containerId;
+ return this;
+ }
public ComponentEvent(String name, ComponentEventType type) {
super(type);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
index 9e5f98c..509f667 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/instance/ComponentInstance.java
@@ -146,7 +146,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
compInstance.containerStatusFuture =
compInstance.scheduler.executorService.scheduleAtFixedRate(
new ContainerStatusRetriever(compInstance.scheduler,
- compInstance.getContainerId(), compInstance), 0, 1,
+ event.getContainerId(), compInstance), 0, 1,
TimeUnit.SECONDS);
compInstance.component.incRunningContainers();
long containerStartTime = System.currentTimeMillis();
@@ -160,10 +160,10 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
org.apache.hadoop.yarn.service.api.records.Container container =
new org.apache.hadoop.yarn.service.api.records.Container();
- container.setId(compInstance.getContainerId().toString());
+ container.setId(event.getContainerId().toString());
container.setLaunchTime(new Date(containerStartTime));
container.setState(ContainerState.RUNNING_BUT_UNREADY);
- container.setBareHost(compInstance.container.getNodeId().getHost());
+ container.setBareHost(compInstance.getNodeId().getHost());
container.setComponentInstanceName(compInstance.getCompInstanceName());
if (compInstance.containerSpec != null) {
// remove the previous container.
@@ -219,15 +219,11 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
// re-ask the failed container.
Component comp = compInstance.component;
comp.requestContainers(1);
- LOG.info(compInstance.getCompInstanceId()
- + ": Container completed. Requested a new container." + System
- .lineSeparator() + " exitStatus={}, diagnostics={}.",
- event.getStatus().getExitStatus(),
- event.getStatus().getDiagnostics());
String containerDiag =
compInstance.getCompInstanceId() + ": " + event.getStatus()
.getDiagnostics();
compInstance.diagnostics.append(containerDiag + System.lineSeparator());
+ compInstance.cancelContainerStatusRetriever();
if (compInstance.getState().equals(READY)) {
compInstance.component.decContainersReady();
@@ -255,11 +251,13 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
// hdfs dir content will be overwritten when a new container gets started,
// so no need remove.
compInstance.scheduler.executorService
- .submit(compInstance::cleanupRegistry);
+ .submit(() -> compInstance.cleanupRegistry(event.getContainerId()));
+
if (compInstance.timelineServiceEnabled) {
// record in ATS
- compInstance.serviceTimelinePublisher.componentInstanceFinished
- (compInstance, event.getStatus().getExitStatus(), containerDiag);
+ compInstance.serviceTimelinePublisher
+ .componentInstanceFinished(event.getContainerId(),
+ event.getStatus().getExitStatus(), containerDiag);
}
compInstance.containerSpec.setState(ContainerState.STOPPED);
}
@@ -267,6 +265,14 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
// remove the failed ContainerId -> CompInstance mapping
comp.getScheduler().removeLiveCompInstance(event.getContainerId());
+ comp.reInsertPendingInstance(compInstance);
+
+ LOG.info(compInstance.getCompInstanceId()
+ + ": {} completed. Reinsert back to pending list and requested " +
+ "a new container." + System.lineSeparator() +
+ " exitStatus={}, diagnostics={}.",
+ event.getContainerId(), event.getStatus().getExitStatus(),
+ event.getStatus().getDiagnostics());
if (shouldExit) {
// Sleep for 5 seconds in hope that the state can be recorded in ATS.
// in case there's a client polling the comp state, it can be notified.
@@ -277,8 +283,6 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
ExitUtil.terminate(-1);
}
-
- compInstance.removeContainer();
}
}
@@ -312,15 +316,6 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
}
- public boolean hasContainer() {
- return this.container != null;
- }
-
- public void removeContainer() {
- this.container = null;
- this.compInstanceId.setContainerId(null);
- }
-
public void setContainer(Container container) {
this.container = container;
this.compInstanceId.setContainerId(container.getId());
@@ -337,7 +332,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
public void updateContainerStatus(ContainerStatus status) {
this.status = status;
org.apache.hadoop.yarn.service.api.records.Container container =
- getCompSpec().getContainer(getContainerId().toString());
+ getCompSpec().getContainer(status.getContainerId().toString());
if (container != null) {
container.setIp(StringUtils.join(",", status.getIPs()));
container.setHostname(status.getHost());
@@ -348,10 +343,6 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
updateServiceRecord(yarnRegistryOperations, status);
}
- public ContainerId getContainerId() {
- return container.getId();
- }
-
public String getCompName() {
return compInstanceId.getCompName();
}
@@ -423,12 +414,7 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
public void destroy() {
LOG.info(getCompInstanceId() + ": Flexed down by user, destroying.");
diagnostics.append(getCompInstanceId() + ": Flexed down by user");
- if (container != null) {
- scheduler.removeLiveCompInstance(container.getId());
- component.getScheduler().getAmRMClient()
- .releaseAssignedContainer(container.getId());
- getCompSpec().removeContainer(containerSpec);
- }
+
// update metrics
if (getState() == STARTED) {
component.decRunningContainers();
@@ -437,16 +423,29 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
component.decContainersReady();
component.decRunningContainers();
}
+ getCompSpec().removeContainer(containerSpec);
+
+ if (container == null) {
+ LOG.info(getCompInstanceId() + " no container is assigned when " +
+ "destroying");
+ return;
+ }
+
+ ContainerId containerId = container.getId();
+ scheduler.removeLiveCompInstance(containerId);
+ component.getScheduler().getAmRMClient()
+ .releaseAssignedContainer(containerId);
if (timelineServiceEnabled) {
- serviceTimelinePublisher.componentInstanceFinished(this,
+ serviceTimelinePublisher.componentInstanceFinished(containerId,
KILLED_BY_APPMASTER, diagnostics.toString());
}
- scheduler.executorService.submit(this::cleanupRegistryAndCompHdfsDir);
+ cancelContainerStatusRetriever();
+ scheduler.executorService.submit(() ->
+ cleanupRegistryAndCompHdfsDir(containerId));
}
- private void cleanupRegistry() {
- ContainerId containerId = getContainerId();
+ private void cleanupRegistry(ContainerId containerId) {
String cid = RegistryPathUtils.encodeYarnID(containerId.toString());
try {
yarnRegistryOperations.deleteComponent(getCompInstanceId(), cid);
@@ -456,8 +455,8 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
//TODO Maybe have a dedicated cleanup service.
- public void cleanupRegistryAndCompHdfsDir() {
- cleanupRegistry();
+ public void cleanupRegistryAndCompHdfsDir(ContainerId containerId) {
+ cleanupRegistry(containerId);
try {
if (compInstanceDir != null && fs.exists(compInstanceDir)) {
boolean deleted = fs.delete(compInstanceDir, true);
@@ -515,6 +514,12 @@ public class ComponentInstance implements EventHandler<ComponentInstanceEvent>,
}
}
+ private void cancelContainerStatusRetriever() {
+ if (containerStatusFuture != null && !containerStatusFuture.isDone()) {
+ containerStatusFuture.cancel(true);
+ }
+ }
+
@Override
public int compareTo(ComponentInstance to) {
long delta = containerStartedTime - to.containerStartedTime;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
index 0e51a62..b9f3a24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/ContainerLaunchService.java
@@ -87,7 +87,7 @@ public class ContainerLaunchService extends AbstractService{
AbstractLauncher launcher = new AbstractLauncher(fs, null);
try {
provider.buildContainerLaunchContext(launcher, service,
- instance, fs, getConfig());
+ instance, fs, getConfig(), container);
instance.getComponent().getScheduler().getNmClient()
.startContainerAsync(container,
launcher.completeContainerLaunch());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
index 6d74061..7015591 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/AbstractProviderService.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.service.provider;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -55,7 +56,7 @@ public abstract class AbstractProviderService implements ProviderService,
public void buildContainerLaunchContext(AbstractLauncher launcher,
Service service, ComponentInstance instance,
- SliderFileSystem fileSystem, Configuration yarnConf)
+ SliderFileSystem fileSystem, Configuration yarnConf, Container container)
throws IOException, SliderException {
Component component = instance.getComponent().getComponentSpec();;
processArtifact(launcher, instance, fileSystem, service);
@@ -67,7 +68,7 @@ public abstract class AbstractProviderService implements ProviderService,
Map<String, String> globalTokens =
instance.getComponent().getScheduler().globalTokens;
Map<String, String> tokensForSubstitution = ProviderUtils
- .initCompTokensForSubstitute(instance);
+ .initCompTokensForSubstitute(instance, container);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
launcher.putEnv(ServiceUtils
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
index eb721b4..11015ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderService.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.service.provider;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
@@ -34,6 +35,6 @@ public interface ProviderService {
*/
void buildContainerLaunchContext(AbstractLauncher containerLauncher,
Service service, ComponentInstance instance,
- SliderFileSystem sliderFileSystem, Configuration yarnConf)
- throws IOException, SliderException;
+ SliderFileSystem sliderFileSystem, Configuration yarnConf, Container
+ container) throws IOException, SliderException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
index e074dd7..c0c44c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/ProviderUtils.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.service.ServiceContext;
@@ -393,13 +394,13 @@ public class ProviderUtils implements YarnServiceConstants {
* @return tokens to replace
*/
public static Map<String, String> initCompTokensForSubstitute(
- ComponentInstance instance) {
+ ComponentInstance instance, Container container) {
Map<String, String> tokens = new HashMap<>();
tokens.put(COMPONENT_NAME, instance.getCompSpec().getName());
tokens
.put(COMPONENT_NAME_LC, instance.getCompSpec().getName().toLowerCase());
tokens.put(COMPONENT_INSTANCE_NAME, instance.getCompInstanceName());
- tokens.put(CONTAINER_ID, instance.getContainer().getId().toString());
+ tokens.put(CONTAINER_ID, container.getId().toString());
tokens.put(COMPONENT_ID,
String.valueOf(instance.getCompInstanceId().getId()));
tokens.putAll(instance.getComponent().getDependencyHostIpTokens());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
index c522986..949ce19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/timelineservice/ServiceTimelinePublisher.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.service.timelineservice;
import org.apache.hadoop.metrics2.AbstractMetric;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@@ -178,10 +179,10 @@ public class ServiceTimelinePublisher extends CompositeService {
putEntity(entity);
}
- public void componentInstanceFinished(ComponentInstance instance,
+ public void componentInstanceFinished(ContainerId containerId,
int exitCode, String diagnostics) {
TimelineEntity entity = createComponentInstanceEntity(
- instance.getContainer().getId().toString());
+ containerId.toString());
// create info keys
Map<String, Object> entityInfos = new HashMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
index d343a03..4298161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -24,14 +24,8 @@ import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
-import org.apache.hadoop.yarn.api.records.Resource;
+
+import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.NMClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -42,15 +36,15 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol;
import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.component.Component;
import org.apache.hadoop.yarn.service.component.ComponentState;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState;
import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.util.Records;
import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
+import java.util.*;
import java.util.concurrent.TimeoutException;
import static org.mockito.Mockito.mock;
@@ -63,6 +57,8 @@ public class MockServiceAM extends ServiceMaster {
final List<Container> feedContainers =
Collections.synchronizedList(new LinkedList<>());
+ final List<ContainerStatus> failedContainers =
+ Collections.synchronizedList(new LinkedList<>());
public MockServiceAM(Service service) {
super(service.getName());
this.service = service;
@@ -102,10 +98,10 @@ public class MockServiceAM extends ServiceMaster {
AllocateResponse.AllocateResponseBuilder builder =
AllocateResponse.newBuilder();
+ // add new containers if any
synchronized (feedContainers) {
if (feedContainers.isEmpty()) {
System.out.println("Allocating........ no containers");
- return builder.build();
} else {
// The AMRMClient will return containers for compoenent that are
// at FLEXING state
@@ -121,9 +117,20 @@ public class MockServiceAM extends ServiceMaster {
itor.remove();
}
}
- return builder.allocatedContainers(allocatedContainers).build();
+ builder.allocatedContainers(allocatedContainers);
+ }
+ }
+
+ // add failed containers if any
+ synchronized (failedContainers) {
+ if (!failedContainers.isEmpty()) {
+ List<ContainerStatus> failed =
+ new LinkedList<>(failedContainers);
+ failedContainers.clear();
+ builder.completedContainersStatuses(failed);
}
}
+ return builder.build();
}
@Override
@@ -184,6 +191,19 @@ public class MockServiceAM extends ServiceMaster {
return container;
}
+ public void feedFailedContainerToComp(Service service, int id, String
+ compName) {
+ ApplicationId applicationId = ApplicationId.fromString(service.getId());
+ ContainerId containerId = ContainerId
+ .newContainerId(ApplicationAttemptId.newInstance(applicationId, 1), id);
+ ContainerStatus containerStatus = Records.newRecord(ContainerStatus.class);
+ containerStatus.setContainerId(containerId);
+ synchronized (failedContainers) {
+ failedContainers.add(containerStatus);
+ }
+ }
+
+
public void flexComponent(String compName, long numberOfContainers)
throws IOException {
ClientAMProtocol.ComponentCountProto componentCountProto =
@@ -218,4 +238,22 @@ public class MockServiceAM extends ServiceMaster {
}
}, 1000, 20000);
}
+
+
+ public ComponentInstance getCompInstance(String compName, String
+ instanceName) {
+ return context.scheduler.getAllComponents().get(compName)
+ .getComponentInstance(instanceName);
+ }
+
+ public void waitForCompInstanceState(ComponentInstance instance,
+ ComponentInstanceState state)
+ throws TimeoutException, InterruptedException {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return instance.getState().equals(state);
+ }
+ }, 1000, 20000);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index cf32880..a70a0c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -65,6 +65,7 @@ public class ServiceTestUtils {
private MiniYARNCluster yarnCluster = null;
private MiniDFSCluster hdfsCluster = null;
+ TestingCluster zkCluster;
private FileSystem fs = null;
private Configuration conf = null;
public static final int NUM_NMS = 1;
@@ -165,7 +166,6 @@ public class ServiceTestUtils {
conf.setBoolean(NM_VMEM_CHECK_ENABLED, false);
conf.setBoolean(NM_PMEM_CHECK_ENABLED, false);
// setup zk cluster
- TestingCluster zkCluster;
zkCluster = new TestingCluster(1);
zkCluster.start();
conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
@@ -239,6 +239,9 @@ public class ServiceTestUtils {
hdfsCluster = null;
}
}
+ if (zkCluster != null) {
+ zkCluster.stop();
+ }
if (basedir != null) {
FileUtils.deleteDirectory(basedir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
new file mode 100644
index 0000000..fb4de0d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceAM.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.curator.test.TestingCluster;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstance;
+import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceState;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.registry.client.api.RegistryConstants
+ .KEY_REGISTRY_ZK_QUORUM;
+
+public class TestServiceAM extends ServiceTestUtils{
+
+ private File basedir;
+ YarnConfiguration conf = new YarnConfiguration();
+ TestingCluster zkCluster;
+
+ @Before
+ public void setup() throws Exception {
+ basedir = new File("target", "apps");
+ if (basedir.exists()) {
+ FileUtils.deleteDirectory(basedir);
+ } else {
+ basedir.mkdirs();
+ }
+ zkCluster = new TestingCluster(1);
+ zkCluster.start();
+ conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
+ System.out.println("ZK cluster: " + zkCluster.getConnectString());
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (basedir != null) {
+ FileUtils.deleteDirectory(basedir);
+ }
+ if (zkCluster != null) {
+ zkCluster.stop();
+ }
+ }
+
+ // Race condition YARN-7486
+ // 1. Allocate 1 container to compa and wait it to be started
+ // 2. Fail this container, and in the meanwhile allocate the 2nd container.
+ // 3. The 2nd container should not be assigned to compa-0 instance, because
+ // the compa-0 instance is not stopped yet.
+ // 4. check compa still has the instance in the pending list.
+ @Test
+ public void testContainerCompleted() throws TimeoutException,
+ InterruptedException {
+ ApplicationId applicationId = ApplicationId.newInstance(123456, 1);
+ Service exampleApp = new Service();
+ exampleApp.setId(applicationId.toString());
+ exampleApp.setName("testContainerCompleted");
+ exampleApp.addComponent(createComponent("compa", 1, "pwd"));
+
+ MockServiceAM am = new MockServiceAM(exampleApp);
+ am.init(conf);
+ am.start();
+
+ ComponentInstance compa0 = am.getCompInstance("compa", "compa-0");
+ // allocate a container
+ am.feedContainerToComp(exampleApp, 1, "compa");
+ am.waitForCompInstanceState(compa0, ComponentInstanceState.STARTED);
+
+ System.out.println("Fail the container 1");
+ // fail the container
+ am.feedFailedContainerToComp(exampleApp, 1, "compa");
+
+ // allocate the second container immediately, this container will not be
+ // assigned to comp instance
+ // because the instance is not yet added to the pending list.
+ am.feedContainerToComp(exampleApp, 2, "compa");
+
+ am.waitForCompInstanceState(compa0, ComponentInstanceState.INIT);
+ // still 1 pending instance
+ Assert.assertEquals(1,
+ am.getComponent("compa").getPendingInstances().size());
+ am.stop();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4d5d202/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
index 0e03a2c..e25d38d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/monitor/TestServiceMonitor.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.yarn.service.monitor;
import org.apache.commons.io.FileUtils;
+import org.apache.curator.test.TestingCluster;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.MockServiceAM;
@@ -37,10 +38,14 @@ import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import static org.apache.hadoop.registry.client.api.RegistryConstants
+ .KEY_REGISTRY_ZK_QUORUM;
+
public class TestServiceMonitor extends ServiceTestUtils {
private File basedir;
YarnConfiguration conf = new YarnConfiguration();
+ TestingCluster zkCluster;
@Before
public void setup() throws Exception {
@@ -51,6 +56,10 @@ public class TestServiceMonitor extends ServiceTestUtils {
basedir.mkdirs();
}
conf.setLong(YarnServiceConf.READINESS_CHECK_INTERVAL, 2);
+ zkCluster = new TestingCluster(1);
+ zkCluster.start();
+ conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
+ System.out.println("ZK cluster: " + zkCluster.getConnectString());
}
@After
@@ -58,6 +67,9 @@ public class TestServiceMonitor extends ServiceTestUtils {
if (basedir != null) {
FileUtils.deleteDirectory(basedir);
}
+ if (zkCluster != null) {
+ zkCluster.stop();
+ }
}
// Create compa with 1 container
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/19] hadoop git commit: YARN-7503. Configurable heap size / JVM
opts in service AM. Contributed by Jonathan Hung
Posted by as...@apache.org.
YARN-7503. Configurable heap size / JVM opts in service AM. Contributed by Jonathan Hung
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bf2c301
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bf2c301
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bf2c301
Branch: refs/heads/YARN-6592
Commit: 6bf2c301924a3acae5a7510b8473f6292a5a471b
Parents: 28d0fcb
Author: Jian He <ji...@apache.org>
Authored: Thu Nov 16 10:53:55 2017 -0800
Committer: Jian He <ji...@apache.org>
Committed: Thu Nov 16 10:53:55 2017 -0800
----------------------------------------------------------------------
.../apache/hadoop/yarn/service/client/ServiceClient.java | 10 +++++-----
.../apache/hadoop/yarn/service/conf/YarnServiceConf.java | 5 +++++
.../service/containerlaunch/JavaCommandLineBuilder.java | 11 +++++------
3 files changed, 15 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index af43f8a..d1b6026 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -559,7 +559,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
Map<String, String> env = addAMEnv();
// create AM CLI
- String cmdStr = buildCommandLine(serviceName, conf, appRootDir, hasAMLog4j);
+ String cmdStr = buildCommandLine(app, conf, appRootDir, hasAMLog4j);
submissionContext.setResource(Resource.newInstance(YarnServiceConf
.getLong(YarnServiceConf.AM_RESOURCE_MEM,
YarnServiceConf.DEFAULT_KEY_AM_RESOURCE_MEM, app.getConfiguration(),
@@ -624,12 +624,12 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
LOG.debug(builder.toString());
}
- private String buildCommandLine(String serviceName, Configuration conf,
+ private String buildCommandLine(Service app, Configuration conf,
Path appRootDir, boolean hasSliderAMLog4j) throws BadConfigException {
JavaCommandLineBuilder CLI = new JavaCommandLineBuilder();
CLI.forceIPv4().headless();
- //TODO CLI.setJVMHeap
- //TODO CLI.addJVMOPTS
+ CLI.setJVMOpts(YarnServiceConf.get(YarnServiceConf.JVM_OPTS, null,
+ app.getConfiguration(), conf));
if (hasSliderAMLog4j) {
CLI.sysprop(SYSPROP_LOG4J_CONFIGURATION, YARN_SERVICE_LOG4J_FILENAME);
CLI.sysprop(SYSPROP_LOG_DIR, ApplicationConstants.LOG_DIR_EXPANSION_VAR);
@@ -637,7 +637,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
CLI.add(ServiceMaster.class.getCanonicalName());
//TODO debugAM CLI.add(Arguments.ARG_DEBUG)
CLI.add("-" + ServiceMaster.YARNFILE_OPTION, new Path(appRootDir,
- serviceName + ".json"));
+ app.getName() + ".json"));
// pass the registry binding
CLI.addConfOptionToCLI(conf, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index a7bd58d..684d980 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -86,6 +86,11 @@ public class YarnServiceConf {
public static final int DEFAULT_READINESS_CHECK_INTERVAL = 30; // seconds
/**
+ * JVM opts.
+ */
+ public static final String JVM_OPTS = "yarn.service.am.java.opts";
+
+ /**
* Get long value for the property. First get from the userConf, if not
* present, get from systemConf.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bf2c301/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
index 7f6cc04..bb47971 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/JavaCommandLineBuilder.java
@@ -48,13 +48,12 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
}
/**
- * Set the size of the heap if a non-empty heap is passed in.
- * @param heap empty string or something like "128M" ,"1G" etc. The value is
- * trimmed.
+ * Set JVM opts.
+ * @param jvmOpts JVM opts
*/
- public void setJVMHeap(String heap) {
- if (ServiceUtils.isSet(heap)) {
- add("-Xmx" + heap.trim());
+ public void setJVMOpts(String jvmOpts) {
+ if (ServiceUtils.isSet(jvmOpts)) {
+ add(jvmOpts);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org