You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by zu...@apache.org on 2024/05/22 07:04:21 UTC
(incubator-uniffle) branch branch-0.9 updated (e807f24f7 -> 601119dda)
This is an automated email from the ASF dual-hosted git repository.
zuston pushed a change to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
from e807f24f7 [#1673] fix(K8S): Fix the deployment of stable version K8S cluster (#1694)
new 9c996ca80 [#1721] fix(coordinator): classCastExpection of boolean->String with yaml style remote client conf (#1722)
new 601119dda [MINOR] fix: Update outdated config: rss.writer.send.check.timeout -> rss.client.send.check.timeout.ms (#1734)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 2 +-
.../java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java | 2 +-
.../java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java | 2 +-
.../org/apache/uniffle/coordinator/conf/YamlClientConfParser.java | 7 ++++++-
.../apache/uniffle/coordinator/conf/YamlClientConfParserTest.java | 5 +++++
coordinator/src/test/resources/dynamicClientConf.yaml | 4 ++++
docs/coordinator_guide.md | 2 +-
7 files changed, 19 insertions(+), 5 deletions(-)
(incubator-uniffle) 02/02: [MINOR] fix: Update outdated config: rss.writer.send.check.timeout -> rss.client.send.check.timeout.ms (#1734)
Posted by zu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 601119ddab68cc118d6a03e94d0b821899520f87
Author: RickyMa <ri...@tencent.com>
AuthorDate: Wed May 22 14:53:36 2024 +0800
[MINOR] fix: Update outdated config: rss.writer.send.check.timeout -> rss.client.send.check.timeout.ms (#1734)
### What changes were proposed in this pull request?
Update outdated config: rss.writer.send.check.timeout -> rss.client.send.check.timeout.ms.
### Why are the changes needed?
The `rss.writer.send.check.timeout` configuration has been renamed to `rss.client.send.check.timeout.ms`, no longer in use.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
No need.
---
README.md | 2 +-
.../test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java | 2 +-
.../test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java | 2 +-
docs/coordinator_guide.md | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index dcab92443..3757bd77f 100644
--- a/README.md
+++ b/README.md
@@ -179,7 +179,7 @@ If you have packaged tgz with hadoop jars, the env of `HADOOP_HOME` is needn't s
rss.coordinator.remote.storage.path hdfs://cluster1/path,hdfs://cluster2/path
rss.writer.require.memory.retryMax 1200
rss.client.retry.max 50
- rss.writer.send.check.timeout 600000
+ rss.client.send.check.timeout.ms 600000
rss.client.read.buffer.size 14m
```
5. start Coordinator
diff --git a/client-spark/spark2/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java b/client-spark/spark2/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
index 8711c483b..193cf4121 100644
--- a/client-spark/spark2/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
+++ b/client-spark/spark2/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
@@ -133,7 +133,7 @@ public class RssShuffleWriterTest {
rssShuffleWriter.checkBlockSendResult(Sets.newHashSet(1L, 2L, 3L));
manager.clearTaskMeta(taskId);
- // case 2: partial blocks aren't sent before spark.rss.writer.send.check.timeout,
+ // case 2: partial blocks aren't sent before spark.rss.client.send.check.timeout.ms,
// Runtime exception will be thrown
manager.addSuccessBlockIds(taskId, Sets.newHashSet(1L, 2L));
Throwable e2 =
diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
index 5ca85eced..9c9726ed7 100644
--- a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
+++ b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
@@ -334,7 +334,7 @@ public class RssShuffleWriterTest {
rssShuffleWriter.checkBlockSendResult(Sets.newHashSet(1L, 2L, 3L));
successBlocks.clear();
- // case 2: partial blocks aren't sent before spark.rss.writer.send.check.timeout,
+ // case 2: partial blocks aren't sent before spark.rss.client.send.check.timeout.ms,
// Runtime exception will be thrown
successBlocks.put("taskId", Sets.newHashSet(1L, 2L));
Throwable e2 =
diff --git a/docs/coordinator_guide.md b/docs/coordinator_guide.md
index b33bbba5a..11bd911b1 100644
--- a/docs/coordinator_guide.md
+++ b/docs/coordinator_guide.md
@@ -58,7 +58,7 @@ This document will introduce how to deploy Uniffle coordinators.
rss.coordinator.remote.storage.path hdfs://cluster1/path,hdfs://cluster2/path
rss.writer.require.memory.retryMax 1200
rss.client.retry.max 100
- rss.writer.send.check.timeout 600000
+ rss.client.send.check.timeout.ms 600000
rss.client.read.buffer.size 14m
```
(incubator-uniffle) 01/02: [#1721] fix(coordinator): classCastExpection of boolean->String with yaml style remote client conf (#1722)
Posted by zu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zuston pushed a commit to branch branch-0.9
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 9c996ca801ccf8fd8671cb8c1296d0a6a4d0e56a
Author: Junfan Zhang <zu...@apache.org>
AuthorDate: Tue May 21 09:59:22 2024 +0800
[#1721] fix(coordinator): classCastExpection of boolean->String with yaml style remote client conf (#1722)
### What changes were proposed in this pull request?
fix classCastExpection of boolean->String with yaml style remote client conf
### Why are the changes needed?
Fix: #1721
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Unit tests
---
.../org/apache/uniffle/coordinator/conf/YamlClientConfParser.java | 7 ++++++-
.../apache/uniffle/coordinator/conf/YamlClientConfParserTest.java | 5 +++++
coordinator/src/test/resources/dynamicClientConf.yaml | 4 ++++
3 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/coordinator/src/main/java/org/apache/uniffle/coordinator/conf/YamlClientConfParser.java b/coordinator/src/main/java/org/apache/uniffle/coordinator/conf/YamlClientConfParser.java
index 84841a3a6..d34a6b683 100644
--- a/coordinator/src/main/java/org/apache/uniffle/coordinator/conf/YamlClientConfParser.java
+++ b/coordinator/src/main/java/org/apache/uniffle/coordinator/conf/YamlClientConfParser.java
@@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.stream.Collectors;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
@@ -58,7 +59,11 @@ public class YamlClientConfParser implements ClientConfParser {
private Map<String, String> parseKVItems(Object confRaw) throws Exception {
if (confRaw instanceof Map) {
- return (Map<String, String>) confRaw;
+ return ((Map<?, ?>) confRaw)
+ .entrySet().stream()
+ .collect(
+ Collectors.toMap(
+ x -> String.valueOf(x.getKey()), x -> String.valueOf(x.getValue())));
}
// todo: currently only xml format is supported
diff --git a/coordinator/src/test/java/org/apache/uniffle/coordinator/conf/YamlClientConfParserTest.java b/coordinator/src/test/java/org/apache/uniffle/coordinator/conf/YamlClientConfParserTest.java
index 489465c06..3319656a1 100644
--- a/coordinator/src/test/java/org/apache/uniffle/coordinator/conf/YamlClientConfParserTest.java
+++ b/coordinator/src/test/java/org/apache/uniffle/coordinator/conf/YamlClientConfParserTest.java
@@ -32,6 +32,11 @@ public class YamlClientConfParserTest {
getClass().getClassLoader().getResource("dynamicClientConf.yaml").openStream());
assertEquals("v1", conf.getRssClientConf().get("k1"));
assertEquals("v2", conf.getRssClientConf().get("k2"));
+ assertEquals("true", conf.getRssClientConf().get("k3"));
+ assertEquals("false", conf.getRssClientConf().get("k4"));
+ assertEquals("1", conf.getRssClientConf().get("k5"));
+ assertEquals("2", conf.getRssClientConf().get("k6"));
+
assertEquals(
"v1,v2,v3", conf.getRemoteStorageInfos().get("hdfs://a-ns01").getConfItems().get("k1"));
assertEquals("v1", conf.getRemoteStorageInfos().get("hdfs://x-ns01").getConfItems().get("k1"));
diff --git a/coordinator/src/test/resources/dynamicClientConf.yaml b/coordinator/src/test/resources/dynamicClientConf.yaml
index 44de0ac7e..f0f5a7dc3 100644
--- a/coordinator/src/test/resources/dynamicClientConf.yaml
+++ b/coordinator/src/test/resources/dynamicClientConf.yaml
@@ -18,6 +18,10 @@
rssClientConf:
k1: v1
k2: v2
+ k3: true
+ k4: false
+ k5: 1
+ k6: 2
remoteStorageInfos:
hdfs://a-ns01: |+