You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ro...@apache.org on 2023/05/30 04:07:38 UTC
[incubator-uniffle] branch master updated: [MINOR] refactor: Fix tez workflow and checkstyle (#911)
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 1189580b [MINOR] refactor: Fix tez workflow and checkstyle (#911)
1189580b is described below
commit 1189580bc478cf4429698246b628868e5d85f319
Author: zhengchenyu <zh...@163.com>
AuthorDate: Tue May 30 12:07:32 2023 +0800
[MINOR] refactor: Fix tez workflow and checkstyle (#911)
### What changes were proposed in this pull request?
Fix tez workflow
### Why are the changes needed?
For now, workflow does not build tez client.
---
.github/workflows/sequential.yml | 2 +-
.../java/org/apache/tez/common/RssTezUtils.java | 32 ++++++++++++----------
.../org/apache/tez/common/RssTezUtilsTest.java | 5 ++--
.../org/apache/tez/common/TezIdHelperTest.java | 6 ++--
4 files changed, 25 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/sequential.yml b/.github/workflows/sequential.yml
index 1ecddeb3..1233b890 100644
--- a/.github/workflows/sequential.yml
+++ b/.github/workflows/sequential.yml
@@ -81,7 +81,7 @@ jobs:
run: mvn -B -fae ${{ inputs.maven-args }} -Pmr,hadoop3.2 | tee -a /tmp/maven.log;
shell: bash
- name: Execute `mvn ${{ inputs.maven-args }} -Ptez`
- run: mvn -B -fae ${{ inputs.maven-args }} -Pmr,hadoop2.8 | tee -a /tmp/maven.log;
+ run: mvn -B -fae ${{ inputs.maven-args }} -Ptez | tee -a /tmp/maven.log;
shell: bash
- name: Summary of failures
if: ${{ failure() && inputs.summary != '' }}
diff --git a/client-tez/src/main/java/org/apache/tez/common/RssTezUtils.java b/client-tez/src/main/java/org/apache/tez/common/RssTezUtils.java
index 69837f73..5eac59e8 100644
--- a/client-tez/src/main/java/org/apache/tez/common/RssTezUtils.java
+++ b/client-tez/src/main/java/org/apache/tez/common/RssTezUtils.java
@@ -117,10 +117,12 @@ public class RssTezUtils {
public static String uniformPartitionHostInfo(Map<Integer, List<ShuffleServerInfo>> map) {
List<String> res = new ArrayList<>();
String tmp;
- Set<Integer> pidSet = map.keySet();
- for (Integer pid : pidSet) {
- for (ShuffleServerInfo shuffleServerInfo : map.get(pid)) {
- tmp = pid + UNDERLINE_DELIMITER + shuffleServerInfo.getHost() + COLON_DELIMITER + shuffleServerInfo.getNettyPort();
+ for (Map.Entry<Integer, List<ShuffleServerInfo>> entry : map.entrySet()) {
+ Integer partitionId = entry.getKey();
+ List<ShuffleServerInfo> shuffleServerInfos = entry.getValue();
+ for (ShuffleServerInfo shuffleServerInfo : shuffleServerInfos) {
+ tmp = partitionId + UNDERLINE_DELIMITER + shuffleServerInfo.getHost() + COLON_DELIMITER
+ + shuffleServerInfo.getNettyPort();
res.add(tmp);
}
}
@@ -149,13 +151,13 @@ public class RssTezUtils {
public static String uniformServerToPartitions(Map<String, List<String>> map) {
List<String> res = new ArrayList<>();
- Set<String> keySet = map.keySet();
- for (String s : keySet) {
- String join = StringUtils.join(map.get(s), UNDERLINE_DELIMITER);
- res.add(s + PLUS_DELIMITER + join);
+ for (Map.Entry<String, List<String>> entry : map.entrySet()) {
+ String server = entry.getKey();
+ List<String> partitions = entry.getValue();
+ String join = StringUtils.join(partitions, UNDERLINE_DELIMITER);
+ res.add(server + PLUS_DELIMITER + join);
}
-
- return StringUtils.join(res,COMMA_DELIMITER);
+ return StringUtils.join(res, COMMA_DELIMITER);
}
public static ApplicationAttemptId getApplicationAttemptId() {
@@ -166,7 +168,7 @@ public class RssTezUtils {
public static String uniqueIdentifierToAttemptId(String uniqueIdentifier) {
if (uniqueIdentifier == null) {
- throw new RuntimeException("uniqueIdentifier should not be null");
+ throw new RssException("uniqueIdentifier should not be null");
}
String[] ids = uniqueIdentifier.split("_");
return StringUtils.join(ids, "_", 0, 7);
@@ -175,22 +177,22 @@ public class RssTezUtils {
public static long getBlockId(long partitionId, long taskAttemptId, int nextSeqNo) {
long attemptId = taskAttemptId >> (Constants.PARTITION_ID_MAX_LENGTH + Constants.TASK_ATTEMPT_ID_MAX_LENGTH);
if (attemptId < 0 || attemptId > MAX_ATTEMPT_ID) {
- throw new RuntimeException("Can't support attemptId [" + attemptId
+ throw new RssException("Can't support attemptId [" + attemptId
+ "], the max value should be " + MAX_ATTEMPT_ID);
}
long atomicInt = (nextSeqNo << MAX_ATTEMPT_LENGTH) + attemptId;
if (atomicInt < 0 || atomicInt > Constants.MAX_SEQUENCE_NO) {
- throw new RuntimeException("Can't support sequence [" + atomicInt
+ throw new RssException("Can't support sequence [" + atomicInt
+ "], the max value should be " + Constants.MAX_SEQUENCE_NO);
}
if (partitionId < 0 || partitionId > Constants.MAX_PARTITION_ID) {
- throw new RuntimeException("Can't support partitionId["
+ throw new RssException("Can't support partitionId["
+ partitionId + "], the max value should be " + Constants.MAX_PARTITION_ID);
}
long taskId = taskAttemptId - (attemptId
<< (Constants.PARTITION_ID_MAX_LENGTH + Constants.TASK_ATTEMPT_ID_MAX_LENGTH));
if (taskId < 0 || taskId > Constants.MAX_TASK_ATTEMPT_ID) {
- throw new RuntimeException("Can't support taskId["
+ throw new RssException("Can't support taskId["
+ taskId + "], the max value should be " + Constants.MAX_TASK_ATTEMPT_ID);
}
return (atomicInt << (Constants.PARTITION_ID_MAX_LENGTH + Constants.TASK_ATTEMPT_ID_MAX_LENGTH))
diff --git a/client-tez/src/test/java/org/apache/tez/common/RssTezUtilsTest.java b/client-tez/src/test/java/org/apache/tez/common/RssTezUtilsTest.java
index faa27ab3..5344e543 100644
--- a/client-tez/src/test/java/org/apache/tez/common/RssTezUtilsTest.java
+++ b/client-tez/src/test/java/org/apache/tez/common/RssTezUtilsTest.java
@@ -14,6 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.tez.common;
import org.apache.hadoop.conf.Configuration;
@@ -22,13 +23,13 @@ import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezTaskAttemptID;
import org.apache.tez.dag.records.TezTaskID;
import org.apache.tez.dag.records.TezVertexID;
+import org.junit.jupiter.api.Test;
+
import org.apache.uniffle.common.exception.RssException;
import org.apache.uniffle.common.util.Constants;
-import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
public class RssTezUtilsTest {
diff --git a/client-tez/src/test/java/org/apache/tez/common/TezIdHelperTest.java b/client-tez/src/test/java/org/apache/tez/common/TezIdHelperTest.java
index 60da1e3a..961f358a 100644
--- a/client-tez/src/test/java/org/apache/tez/common/TezIdHelperTest.java
+++ b/client-tez/src/test/java/org/apache/tez/common/TezIdHelperTest.java
@@ -18,16 +18,18 @@
package org.apache.tez.common;
import org.junit.jupiter.api.Test;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
public class TezIdHelperTest {
@Test
- public void TestTetTaskAttemptId(){
+ public void testTetTaskAttemptId() {
TezIdHelper tezIdHelper = new TezIdHelper();
assertEquals(0, tezIdHelper.getTaskAttemptId(27262976));
assertEquals(1, tezIdHelper.getTaskAttemptId(27262977));
assertEquals(0, RssTezUtils.taskIdStrToTaskId("attempt_1680867852986_0012_1_01_000000_0_10003"));
- assertEquals(tezIdHelper.getTaskAttemptId(27262976), RssTezUtils.taskIdStrToTaskId("attempt_1680867852986_0012_1_01_000000_0_10003"));
+ assertEquals(tezIdHelper.getTaskAttemptId(27262976),
+ RssTezUtils.taskIdStrToTaskId("attempt_1680867852986_0012_1_01_000000_0_10003"));
}
}