[GitHub] sijie commented on a change in pull request #1358: Issue #570: Introduce EntryMemTableWithParallelFlusher

[GitBox <gi...@apache.org>]

sijie commented on a change in pull request #1358: Issue #570: Introduce EntryMemTableWithParallelFlusher
URL: https://github.com/apache/bookkeeper/pull/1358#discussion_r182950687
 
 

 ##########
 File path: bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/LedgerCacheTest.java
 ##########
 @@ -613,4 +655,107 @@ private ByteBuf generateEntry(long ledger, long entry) {
         bb.writeBytes(data);
         return bb;
     }
+
+    @Test(timeout = 60000)
+    public void testEntryMemTableParallelFlush() throws Exception {
+        int gcWaitTime = 1000;
+        ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
+        conf.setGcWaitTime(gcWaitTime);
+        conf.setLedgerDirNames(createAndGetLedgerDirs(1));
+        conf.setLedgerStorageClass(FlushTestSortedLedgerStorage.class.getName());
+        // enable entrylog per ledger
+        conf.setEntryLogPerLedgerEnabled(true);
+
+        Bookie bookie = new Bookie(conf);
+        FlushTestSortedLedgerStorage flushTestSortedLedgerStorage = (FlushTestSortedLedgerStorage) bookie.ledgerStorage;
+        EntryMemTable memTable = flushTestSortedLedgerStorage.memTable;
+
+        /*
+         * this bookie.addEntry call is required. FileInfo for Ledger 1, 2, 3
+         * would be created with this call. without the fileinfo,
+         * 'flushTestSortedLedgerStorage.addEntry' calls will fail because of
+         * BOOKKEEPER-965 change.
+         */
+        bookie.addEntry(generateEntry(1, 1), false, new Bookie.NopWriteCallback(), null, "passwd".getBytes());
+        bookie.addEntry(generateEntry(2, 1), false, new Bookie.NopWriteCallback(), null, "passwd".getBytes());
+        bookie.addEntry(generateEntry(3, 1), false, new Bookie.NopWriteCallback(), null, "passwd".getBytes());
+
+        flushTestSortedLedgerStorage.addEntry(generateEntry(1, 2));
+        flushTestSortedLedgerStorage.addEntry(generateEntry(2, 2));
+        flushTestSortedLedgerStorage.addEntry(generateEntry(3, 2));
+
+        assertTrue("EntryMemTable SnapShot is expected to be empty", memTable.snapshot.isEmpty());
+        assertFalse("EntryMemTable is not expected to be empty", memTable.isEmpty());
+
+        // inject MemTableSizeLimitReached, so entrymemtable will be flushed
+        flushTestSortedLedgerStorage.setInjectMemTableSizeLimitReached(true);
+        flushTestSortedLedgerStorage.addEntry(generateEntry(1, 3));
+        Thread.sleep(1000);
+
+        // since we simulated sizeLimitReached, snapshot should have been created and flushed
+        assertTrue("EntryMemTable SnapShot is expected to be empty", memTable.snapshot.isEmpty());
+        assertEquals("Flusher called", 1, flushTestSortedLedgerStorage.getNumOfTimesFlushSnapshotCalled());
+    }
+
+    @Test(timeout = 60000)
+    public void testEntryMemTableParallelFlushWithFlushException() throws Exception {
+        int gcWaitTime = 1000;
+        ServerConfiguration conf = TestBKConfiguration.newServerConfiguration();
+        conf.setGcWaitTime(gcWaitTime);
+        conf.setLedgerDirNames(createAndGetLedgerDirs(1));
+        conf.setLedgerStorageClass(FlushTestSortedLedgerStorage.class.getName());
+        // enable entrylog per ledger
+        conf.setEntryLogPerLedgerEnabled(true);
+
+        Bookie bookie = new Bookie(conf);
+        FlushTestSortedLedgerStorage flushTestSortedLedgerStorage = (FlushTestSortedLedgerStorage) bookie.ledgerStorage;
+        EntryMemTable memTable = flushTestSortedLedgerStorage.memTable;
+
+        /*
+         * this bookie.addEntry call is required. FileInfo for Ledger 1, 2, 3
+         * would be created with this call. without the fileinfo,
+         * 'flushTestSortedLedgerStorage.addEntry' calls will fail because of
+         * BOOKKEEPER-965 change.
+         */
+        bookie.addEntry(generateEntry(1, 1), false, new Bookie.NopWriteCallback(), null, "passwd".getBytes());
+        bookie.addEntry(generateEntry(2, 1), false, new Bookie.NopWriteCallback(), null, "passwd".getBytes());
+        bookie.addEntry(generateEntry(3, 1), false, new Bookie.NopWriteCallback(), null, "passwd".getBytes());
+
+        flushTestSortedLedgerStorage.addEntry(generateEntry(1, 4));
+        flushTestSortedLedgerStorage.addEntry(generateEntry(2, 4));
+        flushTestSortedLedgerStorage.addEntry(generateEntry(3, 4));
+
+        // inject MemTableSizeLimitReached and FlushException, so entrymemtable flush will fail
+        flushTestSortedLedgerStorage.setInjectMemTableSizeLimitReached(true);
+        flushTestSortedLedgerStorage.setInjectFlushException(true, 1L);
+
+        flushTestSortedLedgerStorage.addEntry(generateEntry(1, 5));
+        Thread.sleep(1000);
 
 Review comment:
   please make this more deterministic rather than just sleeping

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services