[GitHub] [bookkeeper] dlg99 commented on issue #3040: [FLAKY TEST] RocksDB segfaulted during CompactionTest: rocksdb::DBImpl::NewIterator(rocksdb::ReadOptions const&, rocksdb::ColumnFamilyHandle*)

[GitBox <gi...@apache.org>]

dlg99 commented on issue #3040:
URL: https://github.com/apache/bookkeeper/issues/3040#issuecomment-1036508397


   It is tricky to mock / will require otherwise useless refactoring/test injection, but I reproed it with minimal modifications.
   The trick is to force GC thread to create an iterator on already closed PersistentEntryLogMetadataMap/underlying RocksDB. Probably any kind of access to it is enough.
   
   ```
   diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
   index bf00566f1..a1a902a64 100644
   --- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
   +++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/GarbageCollectorThread.java
   @@ -31,6 +31,7 @@ import java.util.concurrent.ScheduledExecutorService;
    import java.util.concurrent.TimeUnit;
    import java.util.concurrent.atomic.AtomicBoolean;
    
   +import java.util.concurrent.atomic.AtomicInteger;
    import java.util.concurrent.atomic.AtomicLong;
    import java.util.function.Supplier;
    
   @@ -474,6 +475,8 @@ public class GarbageCollectorThread extends SafeRunnable {
            });
        }
    
   +    final AtomicInteger steps = new AtomicInteger(0);
   +
        /**
         * Compact entry logs if necessary.
         *
   @@ -495,6 +498,14 @@ public class GarbageCollectorThread extends SafeRunnable {
            MutableLong end = new MutableLong(start);
            MutableLong timeDiff = new MutableLong(0);
    
   +        while (!steps.compareAndSet(1, 2)) {
   +            try {
   +                Thread.sleep(10);
   +            } catch (InterruptedException e) {
   +                e.printStackTrace();
   +            }
   +        }
   +
            entryLogMetaMap.forEach((entryLogId, meta) -> {
                int bucketIndex = calculateUsageIndex(numBuckets, meta.getUsage());
                entryLogUsageBuckets[bucketIndex]++;
   @@ -562,6 +573,7 @@ public class GarbageCollectorThread extends SafeRunnable {
            }
    
            this.running = false;
   +
            // Interrupt GC executor thread
            gcExecutor.shutdownNow();
            try {
   @@ -569,6 +581,14 @@ public class GarbageCollectorThread extends SafeRunnable {
            } catch (Exception e) {
                LOG.warn("Failed to close entryLog metadata-map", e);
            }
   +
   +        while (!steps.compareAndSet(0, 1)) {
   +            Thread.sleep(10);
   +        }
   +
   +        while (!steps.compareAndSet(2, 3)) {
   +            Thread.sleep(10);
   +        }
        }
    
        /**
   ```


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@bookkeeper.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org