You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ro...@apache.org on 2022/11/23 03:26:03 UTC
[incubator-uniffle] branch master updated: [BUG] Fix flaky test of AQESkewedJoinWithLocalOrderTest (#350)
This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 3a8e29bd [BUG] Fix flaky test of AQESkewedJoinWithLocalOrderTest (#350)
3a8e29bd is described below
commit 3a8e29bddf80fafc9ca4d608677e6cf9e492922f
Author: Junfan Zhang <zu...@apache.org>
AuthorDate: Wed Nov 23 11:25:58 2022 +0800
[BUG] Fix flaky test of AQESkewedJoinWithLocalOrderTest (#350)
### What changes were proposed in this pull request?
Fix flaky test of AQESkewedJoinWithLocalOrderTest
1. From the log of https://github.com/apache/incubator-uniffle/actions/runs/3524774587/jobs/5910627949, it looks the `private @beforeAll` identifier is invalid. So remove it
### Why are the changes needed?
Fix flaky test
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
UT
---
.../java/org/apache/uniffle/test/AQESkewedJoinTest.java | 8 ++------
.../uniffle/test/AQESkewedJoinWithLocalOrderTest.java | 16 +---------------
2 files changed, 3 insertions(+), 21 deletions(-)
diff --git a/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinTest.java b/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinTest.java
index 576f1d71..fa432be9 100644
--- a/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinTest.java
+++ b/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinTest.java
@@ -44,12 +44,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
public class AQESkewedJoinTest extends SparkIntegrationTestBase {
@BeforeAll
- private static void setupServers() throws Exception {
+ public static void setupServers() throws Exception {
CoordinatorConf coordinatorConf = getCoordinatorConf();
- Map<String, String> dynamicConf = Maps.newHashMap();
- dynamicConf.put(CoordinatorConf.COORDINATOR_REMOTE_STORAGE_PATH.key(), HDFS_URI + "rss/test");
- dynamicConf.put(RssSparkConfig.RSS_STORAGE_TYPE.key(), StorageType.MEMORY_LOCALFILE_HDFS.name());
- addDynamicConf(coordinatorConf, dynamicConf);
createCoordinatorServer(coordinatorConf);
ShuffleServerConf shuffleServerConf = getShuffleServerConf();
createShuffleServer(shuffleServerConf);
@@ -68,7 +64,7 @@ public class AQESkewedJoinTest extends SparkIntegrationTestBase {
@Override
public void updateSparkConfCustomer(SparkConf sparkConf) {
- sparkConf.set(RssSparkConfig.RSS_STORAGE_TYPE.key(), "HDFS");
+ sparkConf.set(RssSparkConfig.RSS_STORAGE_TYPE.key(), StorageType.MEMORY_LOCALFILE_HDFS.name());
sparkConf.set(RssSparkConfig.RSS_REMOTE_STORAGE_PATH.key(), HDFS_URI + "rss/test");
}
diff --git a/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinWithLocalOrderTest.java b/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinWithLocalOrderTest.java
index 331f5898..5d56fa62 100644
--- a/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinWithLocalOrderTest.java
+++ b/integration-test/spark3/src/test/java/org/apache/uniffle/test/AQESkewedJoinWithLocalOrderTest.java
@@ -19,30 +19,16 @@ package org.apache.uniffle.test;
import org.apache.spark.SparkConf;
import org.apache.spark.shuffle.RssSparkConfig;
-import org.junit.jupiter.api.BeforeAll;
import org.apache.uniffle.common.ShuffleDataDistributionType;
import org.apache.uniffle.common.config.RssClientConf;
-import org.apache.uniffle.coordinator.CoordinatorConf;
-import org.apache.uniffle.server.ShuffleServerConf;
import org.apache.uniffle.storage.util.StorageType;
public class AQESkewedJoinWithLocalOrderTest extends AQESkewedJoinTest {
- @BeforeAll
- public static void setupServers() throws Exception {
- CoordinatorConf coordinatorConf = getCoordinatorConf();
- createCoordinatorServer(coordinatorConf);
- ShuffleServerConf shuffleServerConf = getShuffleServerConf();
- // Use the LOCALFILE storage type to ensure the data will be flushed by local_order mechanism
- shuffleServerConf.setString("rss.storage.type", StorageType.LOCALFILE.name());
- createShuffleServer(shuffleServerConf);
- startServers();
- }
-
@Override
public void updateSparkConfCustomer(SparkConf sparkConf) {
- sparkConf.set(RssSparkConfig.RSS_STORAGE_TYPE.key(), "LOCALFILE");
+ sparkConf.set(RssSparkConfig.RSS_STORAGE_TYPE.key(), StorageType.LOCALFILE.name());
sparkConf.set("spark." + RssClientConf.DATA_DISTRIBUTION_TYPE.key(),
ShuffleDataDistributionType.LOCAL_ORDER.name());
}