You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by GitBox <gi...@apache.org> on 2022/09/02 20:43:38 UTC
[GitHub] [hadoop] goiri commented on a diff in pull request #4846: YARN-11290. Improve Query Condition of FederationStateStore#getApplicationsHomeSubCluster.
goiri commented on code in PR #4846:
URL: https://github.com/apache/hadoop/pull/4846#discussion_r961990524
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java:
##########
@@ -255,14 +261,33 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
@Override
public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
- List<ApplicationHomeSubCluster> result =
- new ArrayList<ApplicationHomeSubCluster>();
- for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
- result
- .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+
+ if (request == null) {
+ throw new YarnException("Missing getApplicationsHomeSubCluster request");
+ }
+
+ List<ApplicationHomeSubCluster> result = new ArrayList<>();
+ List<ApplicationId> applicationIdList =
+ applications.keySet().stream().collect(Collectors.toList());
+
+ SubClusterId requestSubClusterId = request.getSubClusterId();
+ int appCount = 0;
+ for (int i = 0; i < applicationIdList.size(); i++) {
+ if (appCount >= maxAppsInStateStore) {
Review Comment:
Move this to the for condition.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -255,23 +260,41 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
@Override
public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
+
+ if (request == null) {
+ throw new YarnException("Missing getApplicationsHomeSubCluster request");
+ }
+
long start = clock.getTime();
List<ApplicationHomeSubCluster> result = new ArrayList<>();
+ SubClusterId requestSubClusterId = request.getSubClusterId();
+ int appCount = 0;
try {
- for (String child : zkManager.getChildren(appsZNode)) {
+ List<String> childrens = zkManager.getChildren(appsZNode);
+ for (String child : childrens) {
+ if (appCount >= maxAppsInStateStore) {
+ break;
+ }
ApplicationId appId = ApplicationId.fromString(child);
SubClusterId homeSubCluster = getApp(appId);
- ApplicationHomeSubCluster app =
- ApplicationHomeSubCluster.newInstance(appId, homeSubCluster);
+ // If the requestSubClusterId that needs to be filtered in the request
+ // is inconsistent with the SubClusterId in the data, continue to the next round
+ if (requestSubClusterId != null && !requestSubClusterId.equals(homeSubCluster)) {
+ continue;
Review Comment:
reverse the if
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java:
##########
@@ -726,13 +731,23 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
@Override
public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
+
+ if (request == null) {
+ throw new YarnException("Missing getApplicationsHomeSubCluster request");
+ }
+
CallableStatement cstmt = null;
ResultSet rs = null;
- List<ApplicationHomeSubCluster> appsHomeSubClusters =
- new ArrayList<ApplicationHomeSubCluster>();
+ List<ApplicationHomeSubCluster> appsHomeSubClusters = new ArrayList<>();
try {
cstmt = getCallableStatement(CALL_SP_GET_APPLICATIONS_HOME_SUBCLUSTER);
+ cstmt.setInt("limit_IN", maxAppsInStateStore);
+ String homeSubClusterIN = null;;
Review Comment:
Extra ;
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java:
##########
@@ -255,14 +261,33 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
@Override
public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
- List<ApplicationHomeSubCluster> result =
- new ArrayList<ApplicationHomeSubCluster>();
- for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
- result
- .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+
+ if (request == null) {
+ throw new YarnException("Missing getApplicationsHomeSubCluster request");
+ }
+
+ List<ApplicationHomeSubCluster> result = new ArrayList<>();
+ List<ApplicationId> applicationIdList =
+ applications.keySet().stream().collect(Collectors.toList());
+
+ SubClusterId requestSubClusterId = request.getSubClusterId();
+ int appCount = 0;
+ for (int i = 0; i < applicationIdList.size(); i++) {
+ if (appCount >= maxAppsInStateStore) {
+ break;
+ }
+ ApplicationId applicationId = applicationIdList.get(i);
+ SubClusterId subClusterId = applications.get(applicationId);
+ // If the requestSubClusterId that needs to be filtered in the request
+ // is inconsistent with the SubClusterId in the data, continue to the next round
+ if (requestSubClusterId != null && !requestSubClusterId.equals(subClusterId)){
+ continue;
+ }
+ result.add(ApplicationHomeSubCluster.newInstance(applicationId, subClusterId));
+ appCount++;
}
- GetApplicationsHomeSubClusterResponse.newInstance(result);
Review Comment:
This was just overlooked?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java:
##########
@@ -255,14 +261,33 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
@Override
public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
- List<ApplicationHomeSubCluster> result =
- new ArrayList<ApplicationHomeSubCluster>();
- for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
- result
- .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+
+ if (request == null) {
+ throw new YarnException("Missing getApplicationsHomeSubCluster request");
+ }
+
+ List<ApplicationHomeSubCluster> result = new ArrayList<>();
+ List<ApplicationId> applicationIdList =
+ applications.keySet().stream().collect(Collectors.toList());
+
+ SubClusterId requestSubClusterId = request.getSubClusterId();
+ int appCount = 0;
+ for (int i = 0; i < applicationIdList.size(); i++) {
+ if (appCount >= maxAppsInStateStore) {
Review Comment:
Actually it would be good to do a foreach
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/MemoryFederationStateStore.java:
##########
@@ -255,14 +261,33 @@ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
@Override
public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
GetApplicationsHomeSubClusterRequest request) throws YarnException {
- List<ApplicationHomeSubCluster> result =
- new ArrayList<ApplicationHomeSubCluster>();
- for (Entry<ApplicationId, SubClusterId> e : applications.entrySet()) {
- result
- .add(ApplicationHomeSubCluster.newInstance(e.getKey(), e.getValue()));
+
+ if (request == null) {
+ throw new YarnException("Missing getApplicationsHomeSubCluster request");
+ }
+
+ List<ApplicationHomeSubCluster> result = new ArrayList<>();
+ List<ApplicationId> applicationIdList =
+ applications.keySet().stream().collect(Collectors.toList());
+
+ SubClusterId requestSubClusterId = request.getSubClusterId();
+ int appCount = 0;
+ for (int i = 0; i < applicationIdList.size(); i++) {
+ if (appCount >= maxAppsInStateStore) {
+ break;
+ }
+ ApplicationId applicationId = applicationIdList.get(i);
+ SubClusterId subClusterId = applications.get(applicationId);
+ // If the requestSubClusterId that needs to be filtered in the request
+ // is inconsistent with the SubClusterId in the data, continue to the next round
+ if (requestSubClusterId != null && !requestSubClusterId.equals(subClusterId)){
+ continue;
+ }
+ result.add(ApplicationHomeSubCluster.newInstance(applicationId, subClusterId));
Review Comment:
do the reverse if to avoid the continue
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org