(druid) branch master updated: test: poll sys.segments in verifyNumVisibleSegmentsIs to fix flaky compaction tests (#20320)

[fr...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

FrankChen021 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git


The following commit(s) were added to refs/heads/master by this push:
     new afa5b4e7441 test: poll sys.segments in verifyNumVisibleSegmentsIs to fix flaky compaction tests (#20320)
afa5b4e7441 is described below

commit afa5b4e7441e62bf83f0ba294bdb89afa74ae243
Author: Frank Chen <fr...@outlook.com>
AuthorDate: Wed Sep 30 15:16:36 2026 +0800

    test: poll sys.segments in verifyNumVisibleSegmentsIs to fix flaky compaction tests (#20320)
    
    * test: poll sys.segments in verifyNumVisibleSegmentsIs to fix flaky compaction tests
    
    CompactionTaskTest intermittently failed with an empty sys.segments count after
    waiting for a single segment/metadataCache/sync/time event, because an in-flight
    sync can still observe the previous state. Keep the immediate Overlord-side
    assertion and poll the Broker-side sys.segments count with a deadline, reporting
    the last observed result on failure.
    
    * test: wait for current segment IDs after compaction
    
    * test: preserve percent signs in segment ID filter
---
 .../embedded/compact/CompactionTestBase.java       |  4 +-
 .../testing/embedded/EmbeddedClusterApis.java      | 43 +++++++++++++++++-----
 2 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionTestBase.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionTestBase.java
index b714cd00859..aefb9770bab 100644
--- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionTestBase.java
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/CompactionTestBase.java
@@ -97,8 +97,8 @@ public abstract class CompactionTestBase extends EmbeddedClusterTestBase
 
   protected void verifySegmentsCount(int numExpectedSegments)
   {
-    // Ensure that Broker has synced latest segments from the Coordinator
-    broker.latchableEmitter().waitForNextEvent(event -> event.hasMetricName("segment/metadataCache/sync/time"));
+    // The Overlord state is verified immediately; the Broker-side sys.segments view is polled
+    // until it matches, since a single metadata cache sync may still observe the previous state.
     cluster.callApi().verifyNumVisibleSegmentsIs(numExpectedSegments, dataSource, overlord);
   }
 
diff --git a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java
index 8082bced562..a4fb736ee64 100644
--- a/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java
+++ b/services/src/test/java/org/apache/druid/testing/embedded/EmbeddedClusterApis.java
@@ -61,6 +61,7 @@ import java.util.Set;
 import java.util.TreeSet;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 /**
  * Contains various utility methods to interact with an {@link EmbeddedDruidCluster}.
@@ -281,21 +282,43 @@ public class EmbeddedClusterApis implements EmbeddedResource
    */
   public void verifyNumVisibleSegmentsIs(int numExpectedSegments, String dataSource, EmbeddedOverlord overlord)
   {
-    int segmentCount = getVisibleUsedSegments(dataSource, overlord).size();
+    final Set<DataSegment> visibleSegments = getVisibleUsedSegments(dataSource, overlord);
+    final int segmentCount = visibleSegments.size();
     Assertions.assertEquals(
         numExpectedSegments,
         segmentCount,
         "Segment count mismatch"
     );
-    Assertions.assertEquals(
-        String.valueOf(segmentCount),
-        runSql(
-            "SELECT COUNT(*) FROM sys.segments WHERE datasource='%s'"
-            + " AND is_overshadowed = 0 AND is_available = 1",
-            dataSource
-        ),
-        "Segment count mismatch in sys.segments table"
-    );
+
+    // The Broker learns about segment changes asynchronously from the Coordinator, so the
+    // sys.segments table may briefly lag behind the metadata store. Match the segment IDs as well
+    // as the count because compaction can replace segments without changing their number.
+    final String expectedCount = String.valueOf(segmentCount);
+    final String expectedSegmentIds = visibleSegments
+        .stream()
+        .map(segment -> StringUtils.format("'%s'", StringUtils.escapeSql(segment.getId().toString())))
+        .collect(Collectors.joining(", "));
+    final String segmentIdFilter = expectedSegmentIds.isEmpty()
+                                   ? ""
+                                   : " AND segment_id IN (" + expectedSegmentIds + ")";
+    final String sql = "SELECT COUNT(*) FROM sys.segments WHERE datasource='%s'"
+                       + " AND is_overshadowed = 0 AND is_available = 1%s";
+    try {
+      waitForResult(() -> runSql(sql, StringUtils.escapeSql(dataSource), segmentIdFilter), expectedCount::equals)
+          .withTimeoutMillis(60_000)
+          .go();
+    }
+    catch (ISE e) {
+      throw new AssertionError(
+          StringUtils.format(
+              "Segment count mismatch in sys.segments table: expected[%s] segments for datasource[%s]. %s",
+              expectedCount,
+              dataSource,
+              e.getMessage()
+          ),
+          e
+      );
+    }
   }
 
   /**


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org