You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2023/12/14 10:32:07 UTC
(iceberg) branch main updated: Spark: Fix flaky tests which concurrently modify HashSet (#9294)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 5e62e478d8 Spark: Fix flaky tests which concurrently modify HashSet (#9294)
5e62e478d8 is described below
commit 5e62e478d8eda41fc46f441717e4a43e33f2cba1
Author: Manu Zhang <Ow...@gmail.com>
AuthorDate: Thu Dec 14 18:32:02 2023 +0800
Spark: Fix flaky tests which concurrently modify HashSet (#9294)
---
core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java | 2 +-
.../org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java | 2 +-
.../org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java | 2 +-
.../org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java | 2 +-
.../org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java | 2 +-
.../org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java | 2 +-
.../org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java | 2 +-
7 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
index fc3aa6c916..130fbee9f9 100644
--- a/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
+++ b/core/src/test/java/org/apache/iceberg/TestRemoveSnapshots.java
@@ -795,7 +795,7 @@ public class TestRemoveSnapshots extends TableTestBase {
rewriteManifests.addManifest(newManifest);
rewriteManifests.commit();
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
AtomicInteger planThreadsIndex = new AtomicInteger(0);
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
index 25e6f6486a..c8bef4a2a9 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
@@ -207,7 +207,7 @@ public class TestExpireSnapshotsAction extends SparkTestBase {
long t4 = rightAfterSnapshot();
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
index 536dd5febb..c94aec6f5c 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
@@ -271,7 +271,7 @@ public abstract class TestRemoveOrphanFilesAction extends SparkTestBase {
waitUntilAfter(System.currentTimeMillis());
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
index e90626cbda..d5ac1b8064 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
@@ -207,7 +207,7 @@ public class TestExpireSnapshotsAction extends SparkTestBase {
long t4 = rightAfterSnapshot();
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
index 032ccfb14d..a0caacd6d2 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
@@ -270,7 +270,7 @@ public abstract class TestRemoveOrphanFilesAction extends SparkTestBase {
waitUntilAfter(System.currentTimeMillis());
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
index e90626cbda..d5ac1b8064 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestExpireSnapshotsAction.java
@@ -207,7 +207,7 @@ public class TestExpireSnapshotsAction extends SparkTestBase {
long t4 = rightAfterSnapshot();
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
index 032ccfb14d..a0caacd6d2 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/actions/TestRemoveOrphanFilesAction.java
@@ -270,7 +270,7 @@ public abstract class TestRemoveOrphanFilesAction extends SparkTestBase {
waitUntilAfter(System.currentTimeMillis());
- Set<String> deletedFiles = Sets.newHashSet();
+ Set<String> deletedFiles = ConcurrentHashMap.newKeySet();
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);