You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/08/30 22:05:34 UTC
[15/50] [abbrv] hadoop git commit: YARN-5327. API changes required to
support recurring reservations in the YARN ReservationSystem. (Sangeetha Abdu
Jyothi via Subru).
YARN-5327. API changes required to support recurring reservations in the YARN ReservationSystem. (Sangeetha Abdu Jyothi via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b930dc3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b930dc3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b930dc3e
Branch: refs/heads/YARN-2915
Commit: b930dc3ec06afa479a249490976e3e127d201706
Parents: e806db7
Author: Subru Krishnan <su...@apache.org>
Authored: Fri Aug 26 16:58:47 2016 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Fri Aug 26 16:58:47 2016 -0700
----------------------------------------------------------------------
.../yarn/api/records/ReservationDefinition.java | 49 +++++++++++++++-
.../src/main/proto/yarn_protos.proto | 1 +
.../impl/pb/ReservationDefinitionPBImpl.java | 21 ++++++-
.../reservation/ReservationInputValidator.java | 14 +++++
.../TestReservationInputValidator.java | 61 ++++++++++++++++++++
5 files changed, 143 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
index 10f592a..8ef881b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ReservationDefinition.java
@@ -37,13 +37,24 @@ public abstract class ReservationDefinition {
@Public
@Unstable
public static ReservationDefinition newInstance(long arrival, long deadline,
- ReservationRequests reservationRequests, String name) {
+ ReservationRequests reservationRequests, String name,
+ String recurrenceExpression) {
ReservationDefinition rDefinition =
Records.newRecord(ReservationDefinition.class);
rDefinition.setArrival(arrival);
rDefinition.setDeadline(deadline);
rDefinition.setReservationRequests(reservationRequests);
rDefinition.setReservationName(name);
+ rDefinition.setRecurrenceExpression(recurrenceExpression);
+ return rDefinition;
+ }
+
+ @Public
+ @Unstable
+ public static ReservationDefinition newInstance(long arrival, long deadline,
+ ReservationRequests reservationRequests, String name) {
+ ReservationDefinition rDefinition =
+ newInstance(arrival, deadline, reservationRequests, name, "0");
return rDefinition;
}
@@ -134,4 +145,40 @@ public abstract class ReservationDefinition {
@Evolving
public abstract void setReservationName(String name);
+ /**
+ * Get the recurrence of this reservation representing the time period of
+ * the periodic job. Currently, only long values are supported. Later,
+ * support for regular expressions denoting arbitrary recurrence patterns
+ * (e.g., every Tuesday and Thursday) will be added.
+ * Recurrence is represented in milliseconds for periodic jobs.
+ * Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ * are explicitly cancelled and have higher priority than non-periodic jobs
+ * (during initial placement and replanning). Periodic job allocations are
+ * consistent across runs (flexibility in allocation is leveraged only during
+ * initial placement, allocations remain consistent thereafter).
+ *
+ * @return recurrence of this reservation
+ */
+ @Public
+ @Evolving
+ public abstract String getRecurrenceExpression();
+
+ /**
+ * Set the recurrence of this reservation representing the time period of
+ * the periodic job. Currently, only long values are supported. Later,
+ * support for regular expressions denoting arbitrary recurrence patterns
+ * (e.g., every Tuesday and Thursday) will be added.
+ * Recurrence is represented in milliseconds for periodic jobs.
+ * Recurrence is 0 for non-periodic jobs. Periodic jobs are valid until they
+ * are explicitly cancelled and have higher priority than non-periodic jobs
+ * (during initial placement and replanning). Periodic job allocations are
+ * consistent across runs (flexibility in allocation is leveraged only during
+ * initial placement, allocations remain consistent thereafter).
+ *
+ * @param recurrenceExpression recurrence interval of this reservation
+ */
+ @Public
+ @Evolving
+ public abstract void setRecurrenceExpression(String recurrenceExpression);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 8a6ae3f..2cc1784 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -477,6 +477,7 @@ message ReservationDefinitionProto {
optional int64 arrival = 2;
optional int64 deadline = 3;
optional string reservation_name = 4;
+ optional string recurrence_expression = 5 [default = "0"];
}
message ResourceAllocationRequestProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
index cbe7ccd..b30cd2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ReservationDefinitionPBImpl.java
@@ -162,8 +162,9 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition {
@Override
public String toString() {
return "{Arrival: " + getArrival() + ", Deadline: " + getDeadline()
- + ", Reservation Name: " + getReservationName() + ", Resources: "
- + getReservationRequests() + "}";
+ + ", Reservation Name: " + getReservationName()
+ + ", Recurrence expression: " + getRecurrenceExpression()
+ + ", Resources: " + getReservationRequests() + "}";
}
@Override
@@ -181,4 +182,20 @@ public class ReservationDefinitionPBImpl extends ReservationDefinition {
return false;
}
+ @Override
+ public String getRecurrenceExpression() {
+ ReservationDefinitionProtoOrBuilder p = viaProto ? proto : builder;
+ if (p.hasRecurrenceExpression()) {
+ String recurrenceExpression = p.getRecurrenceExpression();
+ return recurrenceExpression;
+ } else {
+ return "0";
+ }
+ }
+
+ @Override
+ public void setRecurrenceExpression(String recurrenceExpression) {
+ builder.setRecurrenceExpression(recurrenceExpression);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
index 8ebde0a..068894f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationInputValidator.java
@@ -155,6 +155,20 @@ public class ReservationInputValidator {
"validate reservation input definition", "ClientRMService", message);
throw RPCUtil.getRemoteException(message);
}
+ // check that the recurrence is a positive long value.
+ String recurrenceExpression = contract.getRecurrenceExpression();
+ try {
+ Long recurrence = Long.parseLong(recurrenceExpression);
+ if (recurrence < 0) {
+ message = "Negative Period : " + recurrenceExpression + ". Please try"
+ + " again with a non-negative long value as period";
+ throw RPCUtil.getRemoteException(message);
+ }
+ } catch (NumberFormatException e) {
+ message = "Invalid period " + recurrenceExpression + ". Please try"
+ + " again with a non-negative long value as period";
+ throw RPCUtil.getRemoteException(message);
+ }
}
private Plan getPlanFromQueue(ReservationSystem reservationSystem, String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b930dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
index bd40ccd..031a767 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestReservationInputValidator.java
@@ -270,6 +270,59 @@ public class TestReservationInputValidator {
}
@Test
+ public void testSubmitReservationValidRecurrenceExpression() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3, "600000");
+ plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ } catch (YarnException e) {
+ Assert.fail(e.getMessage());
+ }
+ Assert.assertNotNull(plan);
+ }
+
+ @Test
+ public void testSubmitReservationNegativeRecurrenceExpression() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3, "-1234");
+ plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("Negative Period : "));
+ LOG.info(message);
+ }
+ }
+
+ @Test
+ public void testSubmitReservationInvalidRecurrenceExpression() {
+ ReservationSubmissionRequest request =
+ createSimpleReservationSubmissionRequest(1, 1, 1, 5, 3, "123abc");
+ plan = null;
+ try {
+ plan =
+ rrValidator.validateReservationSubmissionRequest(rSystem, request,
+ ReservationSystemTestUtil.getNewReservationId());
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertNull(plan);
+ String message = e.getMessage();
+ Assert.assertTrue(message
+ .startsWith("Invalid period "));
+ LOG.info(message);
+ }
+ }
+
+ @Test
public void testUpdateReservationNormal() {
ReservationUpdateRequest request =
createSimpleReservationUpdateRequest(1, 1, 1, 5, 3);
@@ -625,12 +678,20 @@ public class TestReservationInputValidator {
private ReservationSubmissionRequest createSimpleReservationSubmissionRequest(
int numRequests, int numContainers, long arrival, long deadline,
long duration) {
+ return createSimpleReservationSubmissionRequest(numRequests, numContainers,
+ arrival, deadline, duration, "0");
+ }
+
+ private ReservationSubmissionRequest createSimpleReservationSubmissionRequest(
+ int numRequests, int numContainers, long arrival, long deadline,
+ long duration, String recurrence) {
// create a request with a single atomic ask
ReservationSubmissionRequest request =
new ReservationSubmissionRequestPBImpl();
ReservationDefinition rDef = new ReservationDefinitionPBImpl();
rDef.setArrival(arrival);
rDef.setDeadline(deadline);
+ rDef.setRecurrenceExpression(recurrence);
if (numRequests > 0) {
ReservationRequests reqs = new ReservationRequestsPBImpl();
rDef.setReservationRequests(reqs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org