You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ro...@apache.org on 2022/11/18 02:06:02 UTC
[incubator-uniffle] branch master updated: [MINOR] Migrate RankValue to the package of the common class (#265)
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 2df7594e [MINOR] Migrate RankValue to the package of the common class (#265)
2df7594e is described below
commit 2df7594e4cd5f6195ea27b07d954e24e303e609d
Author: jokercurry <84...@users.noreply.github.com>
AuthorDate: Fri Nov 18 10:05:56 2022 +0800
[MINOR] Migrate RankValue to the package of the common class (#265)
### What changes were proposed in this pull request?
More standardized, and the class can be extended in the future.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
No need.
---
.../coordinator/AbstractSelectStorageStrategy.java | 6 +-
.../AppBalanceSelectStorageStrategy.java | 1 -
.../uniffle/coordinator/ApplicationManager.java | 1 -
.../LowestIOSampleCostSelectStorageStrategy.java | 54 ----------------
.../org/apache/uniffle/coordinator/RankValue.java | 74 ++++++++++++++++++++++
5 files changed, 76 insertions(+), 60 deletions(-)
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java
index d76ceea3..9025ddb4 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/AbstractSelectStorageStrategy.java
@@ -28,8 +28,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.uniffle.common.exception.RssException;
-import static org.apache.uniffle.coordinator.LowestIOSampleCostSelectStorageStrategy.RankValue;
-
/**
* This is a simple implementation class, which provides some methods to check whether the path is normal
*/
@@ -37,11 +35,11 @@ public abstract class AbstractSelectStorageStrategy implements SelectStorageStra
/**
* store remote path -> application count for assignment strategy
*/
- protected final Map<String, LowestIOSampleCostSelectStorageStrategy.RankValue> remoteStoragePathRankValue;
+ protected final Map<String, RankValue> remoteStoragePathRankValue;
protected final int fileSize;
public AbstractSelectStorageStrategy(
- Map<String, LowestIOSampleCostSelectStorageStrategy.RankValue> remoteStoragePathRankValue,
+ Map<String, RankValue> remoteStoragePathRankValue,
CoordinatorConf conf) {
this.remoteStoragePathRankValue = remoteStoragePathRankValue;
fileSize = conf.getInteger(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_SCHEDULE_FILE_SIZE);
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java
index 764b854f..4f6f6535 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/AppBalanceSelectStorageStrategy.java
@@ -35,7 +35,6 @@ import org.slf4j.LoggerFactory;
import org.apache.uniffle.common.RemoteStorageInfo;
import org.apache.uniffle.common.filesystem.HadoopFilesystemProvider;
-import org.apache.uniffle.coordinator.LowestIOSampleCostSelectStorageStrategy.RankValue;
/**
* AppBalanceSelectStorageStrategy will consider the number of apps allocated on each remote path is balanced.
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java
index 0b79b506..71561af9 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/ApplicationManager.java
@@ -38,7 +38,6 @@ import org.slf4j.LoggerFactory;
import org.apache.uniffle.common.RemoteStorageInfo;
import org.apache.uniffle.common.util.Constants;
import org.apache.uniffle.common.util.ThreadUtils;
-import org.apache.uniffle.coordinator.LowestIOSampleCostSelectStorageStrategy.RankValue;
public class ApplicationManager {
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java
index 89172334..551b8f4c 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/LowestIOSampleCostSelectStorageStrategy.java
@@ -21,7 +21,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
@@ -134,57 +133,4 @@ public class LowestIOSampleCostSelectStorageStrategy extends AbstractSelectStora
LOG.warn("No remote storage is available, we will default to the first.");
return availableRemoteStorageInfo.values().iterator().next();
}
-
- static class RankValue {
- AtomicLong costTime;
- AtomicInteger appNum;
- AtomicBoolean isHealthy;
-
- RankValue(int appNum) {
- this.costTime = new AtomicLong(0);
- this.appNum = new AtomicInteger(appNum);
- this.isHealthy = new AtomicBoolean(true);
- }
-
- RankValue(long costTime, int appNum) {
- this.costTime = new AtomicLong(costTime);
- this.appNum = new AtomicInteger(appNum);
- this.isHealthy = new AtomicBoolean(true);
- }
-
- public AtomicLong getCostTime() {
- return costTime;
- }
-
- public AtomicInteger getAppNum() {
- return appNum;
- }
-
- public AtomicBoolean getHealthy() {
- return isHealthy;
- }
-
- public void setCostTime(AtomicLong readAndWriteTime) {
- this.costTime = readAndWriteTime;
- }
-
- public void setAppNum(AtomicInteger appNum) {
- this.appNum = appNum;
- }
-
- public void setHealthy(AtomicBoolean isHealthy) {
- this.isHealthy = isHealthy;
- if (!isHealthy.get()) {
- this.costTime.set(Long.MAX_VALUE);
- }
- }
-
- @Override
- public String toString() {
- return "RankValue{"
- + "costTime=" + costTime
- + ", appNum=" + appNum
- + '}';
- }
- }
}
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/RankValue.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/RankValue.java
new file mode 100644
index 00000000..349698b1
--- /dev/null
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/RankValue.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.uniffle.coordinator;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * The basis for app to select remote storage ranking
+ */
+public class RankValue {
+ AtomicLong costTime;
+ AtomicInteger appNum;
+ AtomicBoolean isHealthy;
+
+ public RankValue(int appNum) {
+ this.costTime = new AtomicLong(0);
+ this.appNum = new AtomicInteger(appNum);
+ this.isHealthy = new AtomicBoolean(true);
+ }
+
+ public RankValue(long costTime, int appNum) {
+ this.costTime = new AtomicLong(costTime);
+ this.appNum = new AtomicInteger(appNum);
+ this.isHealthy = new AtomicBoolean(true);
+ }
+
+ public AtomicLong getCostTime() {
+ return costTime;
+ }
+
+ public AtomicInteger getAppNum() {
+ return appNum;
+ }
+
+ public AtomicBoolean getHealthy() {
+ return isHealthy;
+ }
+
+ public void setCostTime(AtomicLong readAndWriteTime) {
+ this.costTime = readAndWriteTime;
+ }
+
+ public void setHealthy(AtomicBoolean isHealthy) {
+ this.isHealthy = isHealthy;
+ if (!isHealthy.get()) {
+ this.costTime.set(Long.MAX_VALUE);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "RankValue{"
+ + "costTime=" + costTime
+ + ", appNum=" + appNum
+ + ", isHealthy=" + isHealthy + '}';
+ }
+}