You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by GitBox <gi...@apache.org> on 2022/08/23 19:00:08 UTC

[GitHub] [hadoop] goiri commented on a diff in pull request #4764: YARN-11177. Support getNewReservation, submit / update/ Reservation API's for Federation.

goiri commented on code in PR #4764:
URL: https://github.com/apache/hadoop/pull/4764#discussion_r953004392


##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java:
##########
@@ -888,13 +890,120 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
   @Override
   public GetNewReservationResponse getNewReservation(
       GetNewReservationRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null) {
+      routerMetrics.incrGetNewReservationFailedRetrieved();
+      String errMsg = "Missing getNewReservation request.";
+      RouterServerUtil.logAndThrowException(errMsg, null);
+    }
+
+    long startTime = clock.getTime();
+    Map<SubClusterId, SubClusterInfo> subClustersActive =
+        federationFacade.getSubClusters(true);
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+      SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive);
+      LOG.info("getNewReservation try #{} on SubCluster {}.", i, subClusterId);
+      ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+      GetNewReservationResponse response = null;
+      try {
+        response = clientRMProxy.getNewReservation(request);
+      } catch (Exception e) {
+        LOG.warn("Unable to create a new Reservation in SubCluster {}.", subClusterId.getId(), e);
+        subClustersActive.remove(subClusterId);
+      }
+
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetNewReservationRetrieved(stopTime - startTime);
+        return response;
+      }
+    }
+
+    routerMetrics.incrGetNewReservationFailedRetrieved();
+    String errMsg = "Failed to create a new reservation.";
+    throw new YarnException(errMsg);
   }
 
   @Override
   public ReservationSubmissionResponse submitReservation(
       ReservationSubmissionRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null || request.getReservationId() == null
+            || request.getReservationDefinition() == null || request.getQueue() == null) {
+      routerMetrics.incrSubmitReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing submitReservation request or reservationId " +
+               "or reservation definition or queue.", null);
+    }
+
+    ReservationSubmissionResponse response = null;

Review Comment:
   Declare this where we use it.



##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java:
##########
@@ -888,13 +890,120 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
   @Override
   public GetNewReservationResponse getNewReservation(
       GetNewReservationRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null) {
+      routerMetrics.incrGetNewReservationFailedRetrieved();
+      String errMsg = "Missing getNewReservation request.";
+      RouterServerUtil.logAndThrowException(errMsg, null);
+    }
+
+    long startTime = clock.getTime();
+    Map<SubClusterId, SubClusterInfo> subClustersActive =
+        federationFacade.getSubClusters(true);
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+      SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive);
+      LOG.info("getNewReservation try #{} on SubCluster {}.", i, subClusterId);
+      ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+      GetNewReservationResponse response = null;
+      try {
+        response = clientRMProxy.getNewReservation(request);
+      } catch (Exception e) {
+        LOG.warn("Unable to create a new Reservation in SubCluster {}.", subClusterId.getId(), e);
+        subClustersActive.remove(subClusterId);
+      }
+
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetNewReservationRetrieved(stopTime - startTime);
+        return response;
+      }
+    }
+
+    routerMetrics.incrGetNewReservationFailedRetrieved();
+    String errMsg = "Failed to create a new reservation.";
+    throw new YarnException(errMsg);
   }
 
   @Override
   public ReservationSubmissionResponse submitReservation(
       ReservationSubmissionRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null || request.getReservationId() == null
+            || request.getReservationDefinition() == null || request.getQueue() == null) {
+      routerMetrics.incrSubmitReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing submitReservation request or reservationId " +
+               "or reservation definition or queue.", null);
+    }
+
+    ReservationSubmissionResponse response = null;
+    long startTime = clock.getTime();
+    ReservationId reservationId = request.getReservationId();
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+
+      SubClusterId subClusterId = policyFacade.getReservationHomeSubCluster(request);
+
+      LOG.info("submitReservation reservationId {} try #{} on SubCluster {}.",
+           reservationId, i, subClusterId);
+
+      ReservationHomeSubCluster reservationHomeSubCluster =
+          ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+
+      if (i == 0) {
+        try {
+          // persist the mapping of reservationId and the subClusterId which has
+          // been selected as its home
+          subClusterId = federationFacade.addReservationHomeSubCluster(reservationHomeSubCluster);
+        } catch (YarnException e) {
+          routerMetrics.incrSubmitReservationFailedRetrieved();
+          String message =
+              String.format("Unable to insert the ReservationId %s into the FederationStateStore.",
+                  reservationId);
+          RouterServerUtil.logAndThrowException(message, e);
+        }
+      } else {
+        try {
+          // update the mapping of reservationId and the home subClusterId to
+          // the new subClusterId we have selected
+          federationFacade.updateReservationHomeSubCluster(reservationHomeSubCluster);
+        } catch (YarnException e) {
+          String message =
+              String.format("Unable to update the ReservationId %s into the FederationStateStore.",

Review Comment:
   We had some utility for the log and throw that did some of this message formatting



##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java:
##########
@@ -888,13 +890,120 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
   @Override
   public GetNewReservationResponse getNewReservation(
       GetNewReservationRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null) {
+      routerMetrics.incrGetNewReservationFailedRetrieved();
+      String errMsg = "Missing getNewReservation request.";
+      RouterServerUtil.logAndThrowException(errMsg, null);
+    }
+
+    long startTime = clock.getTime();
+    Map<SubClusterId, SubClusterInfo> subClustersActive =
+        federationFacade.getSubClusters(true);
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+      SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive);
+      LOG.info("getNewReservation try #{} on SubCluster {}.", i, subClusterId);
+      ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+      GetNewReservationResponse response = null;
+      try {
+        response = clientRMProxy.getNewReservation(request);
+      } catch (Exception e) {
+        LOG.warn("Unable to create a new Reservation in SubCluster {}.", subClusterId.getId(), e);
+        subClustersActive.remove(subClusterId);
+      }
+
+      if (response != null) {
+        long stopTime = clock.getTime();
+        routerMetrics.succeededGetNewReservationRetrieved(stopTime - startTime);
+        return response;
+      }
+    }
+
+    routerMetrics.incrGetNewReservationFailedRetrieved();
+    String errMsg = "Failed to create a new reservation.";
+    throw new YarnException(errMsg);
   }
 
   @Override
   public ReservationSubmissionResponse submitReservation(
       ReservationSubmissionRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null || request.getReservationId() == null
+            || request.getReservationDefinition() == null || request.getQueue() == null) {
+      routerMetrics.incrSubmitReservationFailedRetrieved();
+      RouterServerUtil.logAndThrowException(
+          "Missing submitReservation request or reservationId " +
+               "or reservation definition or queue.", null);
+    }
+
+    ReservationSubmissionResponse response = null;
+    long startTime = clock.getTime();
+    ReservationId reservationId = request.getReservationId();
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+
+      SubClusterId subClusterId = policyFacade.getReservationHomeSubCluster(request);
+
+      LOG.info("submitReservation reservationId {} try #{} on SubCluster {}.",
+           reservationId, i, subClusterId);
+
+      ReservationHomeSubCluster reservationHomeSubCluster =
+          ReservationHomeSubCluster.newInstance(reservationId, subClusterId);
+
+      if (i == 0) {

Review Comment:
   There has to be a cleaner way to do this



##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java:
##########
@@ -888,13 +890,120 @@ public MoveApplicationAcrossQueuesResponse moveApplicationAcrossQueues(
   @Override
   public GetNewReservationResponse getNewReservation(
       GetNewReservationRequest request) throws YarnException, IOException {
-    throw new NotImplementedException("Code is not implemented");
+
+    if (request == null) {
+      routerMetrics.incrGetNewReservationFailedRetrieved();
+      String errMsg = "Missing getNewReservation request.";
+      RouterServerUtil.logAndThrowException(errMsg, null);
+    }
+
+    long startTime = clock.getTime();
+    Map<SubClusterId, SubClusterInfo> subClustersActive =
+        federationFacade.getSubClusters(true);
+
+    for (int i = 0; i < numSubmitRetries; ++i) {
+      SubClusterId subClusterId = getRandomActiveSubCluster(subClustersActive);
+      LOG.info("getNewReservation try #{} on SubCluster {}.", i, subClusterId);
+      ApplicationClientProtocol clientRMProxy = getClientRMProxyForSubCluster(subClusterId);
+      GetNewReservationResponse response = null;
+      try {
+        response = clientRMProxy.getNewReservation(request);
+      } catch (Exception e) {
+        LOG.warn("Unable to create a new Reservation in SubCluster {}.", subClusterId.getId(), e);
+        subClustersActive.remove(subClusterId);
+      }
+
+      if (response != null) {
+        long stopTime = clock.getTime();

Review Comment:
   Can we do this inside of the try?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org