[ignite] branch master updated: IGNITE-14458 Fix flaky IgniteLocalWalSizeTest. (#8958)

[ib...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ibessonov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git


The following commit(s) were added to refs/heads/master by this push:
     new b014376  IGNITE-14458 Fix flaky IgniteLocalWalSizeTest. (#8958)
b014376 is described below

commit b014376d80b8bb7a8e01782130c92757308aeb41
Author: Kirill Tkalenko <tk...@yandex.ru>
AuthorDate: Fri Apr 2 10:58:34 2021 +0300

    IGNITE-14458 Fix flaky IgniteLocalWalSizeTest. (#8958)
---
 .../IgniteDataStorageMetricsSelfTest.java          | 11 -------
 .../persistence/db/wal/IgniteLocalWalSizeTest.java | 31 +++++++++++++-----
 .../db/wal/WriteAheadLogManagerSelfTest.java       | 10 ------
 .../persistence/wal/WalArchiveConsistencyTest.java | 10 ------
 .../junits/common/GridCommonAbstractTest.java      | 38 ++++++++++++++++++++++
 5 files changed, 60 insertions(+), 40 deletions(-)

diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
index 0be8220..eea82cf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgniteDataStorageMetricsSelfTest.java
@@ -43,7 +43,6 @@ import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.metric.MetricRegistry;
 import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric;
 import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
@@ -441,16 +440,6 @@ public class IgniteDataStorageMetricsSelfTest extends GridCommonAbstractTest {
     }
 
     /**
-     * Getting WAL manger.
-     *
-     * @param n Node.
-     * @return WAL manager.
-     */
-    private FileWriteAheadLogManager walMgr(IgniteEx n) {
-        return (FileWriteAheadLogManager)n.context().cache().context().wal();
-    }
-
-    /**
      * Getting db manager of node.
      *
      * @param n Node.
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteLocalWalSizeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteLocalWalSizeTest.java
index 2854a2a..d2244f1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteLocalWalSizeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteLocalWalSizeTest.java
@@ -31,7 +31,6 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
-import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
 import org.apache.ignite.internal.processors.cache.persistence.wal.filehandle.FileWriteHandle;
 import org.apache.ignite.internal.util.typedef.F;
 import org.apache.ignite.internal.util.typedef.internal.U;
@@ -39,11 +38,13 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
 import org.jetbrains.annotations.Nullable;
 import org.junit.Test;
 
+import static org.apache.ignite.configuration.DataStorageConfiguration.UNLIMITED_WAL_ARCHIVE;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.ZIP_SUFFIX;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor.fileName;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER;
 import static org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager.isSegmentFileName;
 import static org.apache.ignite.testframework.GridTestUtils.getFieldValue;
+import static org.apache.ignite.testframework.GridTestUtils.waitForCondition;
 
 /**
  * Class for testing local size of WAL.
@@ -73,6 +74,7 @@ public class IgniteLocalWalSizeTest extends GridCommonAbstractTest {
                 new DataStorageConfiguration()
                     .setWalSegments(5)
                     .setWalSegmentSize((int)U.MB)
+                    .setMaxWalArchiveSize(UNLIMITED_WAL_ARCHIVE)
                     .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))
             );
     }
@@ -185,12 +187,23 @@ public class IgniteLocalWalSizeTest extends GridCommonAbstractTest {
      * Check that local segment sizes in the memory and actual match.
      *
      * @param n Node.
+     * @throws Exception If failed.
      */
-    private void checkLocalSegmentSizes(IgniteEx n) {
-        FileWriteAheadLogManager wal = (FileWriteAheadLogManager)n.context().cache().context().wal();
+    private void checkLocalSegmentSizes(IgniteEx n) throws Exception {
+        disableWal(n, true);
+
+        if (walMgr(n).getSegmentRouter().hasArchive()) {
+            assertTrue(waitForCondition(
+                () -> walMgr(n).lastArchivedSegment() == walMgr(n).currentSegment() - 1, getTestTimeout()));
+        }
+
+        if (n.context().config().getDataStorageConfiguration().isWalCompactionEnabled()) {
+            assertTrue(waitForCondition(
+                () -> walMgr(n).lastCompactedSegment() == walMgr(n).lastArchivedSegment(), getTestTimeout()));
+        }
 
-        File walWorkDir = getFieldValue(wal, "walWorkDir");
-        File walArchiveDir = getFieldValue(wal, "walArchiveDir");
+        File walWorkDir = getFieldValue(walMgr(n), "walWorkDir");
+        File walArchiveDir = getFieldValue(walMgr(n), "walArchiveDir");
 
         Map<Long, Long> expSegmentSize = new HashMap<>();
 
@@ -204,7 +217,7 @@ public class IgniteLocalWalSizeTest extends GridCommonAbstractTest {
                     expSegmentSize.putIfAbsent(fd.idx(), fd.file().length());
             });
 
-        FileWriteHandle currHnd = getFieldValue(wal, "currHnd");
+        FileWriteHandle currHnd = getFieldValue(walMgr(n), "currHnd");
 
         if (!walArchiveDir.equals(walWorkDir)) {
             long absIdx = currHnd.getSegmentId();
@@ -216,14 +229,14 @@ public class IgniteLocalWalSizeTest extends GridCommonAbstractTest {
 
         assertEquals(currHnd.getSegmentId() + 1, expSegmentSize.size());
 
-        Map<Long, Long> segmentSize = getFieldValue(wal, "segmentSize");
+        Map<Long, Long> segmentSize = getFieldValue(walMgr(n), "segmentSize");
         assertEquals(expSegmentSize.size(), segmentSize.size());
 
         expSegmentSize.forEach((idx, size) -> {
             assertEquals(idx.toString(), size, segmentSize.get(idx));
-            assertEquals(idx.toString(), size.longValue(), wal.segmentSize(idx));
+            assertEquals(idx.toString(), size.longValue(), walMgr(n).segmentSize(idx));
         });
 
-        assertEquals(0, wal.segmentSize(currHnd.getSegmentId() + 1));
+        assertEquals(0, walMgr(n).segmentSize(currHnd.getSegmentId() + 1));
     }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WriteAheadLogManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WriteAheadLogManagerSelfTest.java
index e978e77..cff984a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WriteAheadLogManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WriteAheadLogManagerSelfTest.java
@@ -299,14 +299,4 @@ public class WriteAheadLogManagerSelfTest extends GridCommonAbstractTest {
     @Nullable private GridTimeoutObject timeoutRollover(IgniteEx n) {
         return getFieldValue(walMgr(n), "timeoutRollover");
     }
-
-    /**
-     * Getting WAL manager.
-     *
-     * @param n Node.
-     * @return WAL manager.
-     */
-    private FileWriteAheadLogManager walMgr(IgniteEx n) {
-        return (FileWriteAheadLogManager)n.context().cache().context().wal();
-    }
 }
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalArchiveConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalArchiveConsistencyTest.java
index 43a26c2..a060eed 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalArchiveConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/WalArchiveConsistencyTest.java
@@ -273,16 +273,6 @@ public class WalArchiveConsistencyTest extends GridCommonAbstractTest {
     }
 
     /**
-     * Getting WAL manager of node.
-     *
-     * @param n Node.
-     * @return WAL manager.
-     */
-    private FileWriteAheadLogManager walMgr(IgniteEx n) {
-        return (FileWriteAheadLogManager)n.context().cache().context().wal();
-    }
-
-    /**
      * Getting db manager of node.
      *
      * @param n Node.
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
index e8a2aa7..95aea5b 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/common/GridCommonAbstractTest.java
@@ -77,6 +77,7 @@ import org.apache.ignite.internal.IgniteEx;
 import org.apache.ignite.internal.IgniteInternalFuture;
 import org.apache.ignite.internal.IgniteInterruptedCheckedException;
 import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
 import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
 import org.apache.ignite.internal.processors.affinity.GridAffinityFunctionContextImpl;
 import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
@@ -90,6 +91,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
 import org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl;
 import org.apache.ignite.internal.processors.cache.PartitionUpdateCounter;
+import org.apache.ignite.internal.processors.cache.WalStateManager.WALDisableContext;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
 import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
 import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtColocatedCache;
@@ -103,6 +105,8 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCach
 import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
 import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
 import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
 import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
 import org.apache.ignite.internal.processors.cache.verify.IdleVerifyResultV2;
@@ -136,6 +140,7 @@ import org.apache.ignite.transactions.TransactionRollbackException;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
+import static java.util.Collections.emptyList;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toSet;
 import static org.apache.ignite.IgniteSystemProperties.IGNITE_EVENT_DRIVEN_SERVICE_PROCESSOR_ENABLED;
@@ -146,6 +151,7 @@ import static org.apache.ignite.configuration.IgniteConfiguration.DFLT_SNAPSHOT_
 import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled;
 import static org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState.OWNING;
 import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+import static org.apache.ignite.testframework.GridTestUtils.setFieldValue;
 import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
 import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
 
@@ -2609,4 +2615,36 @@ public abstract class GridCommonAbstractTest extends GridAbstractTest {
             }
         }
     }
+
+     /**
+      * Getting WAL manager.
+      *
+      * @param n Node.
+      * @return WAL manger.
+     */
+     protected static FileWriteAheadLogManager walMgr(IgniteEx n) {
+         return (FileWriteAheadLogManager)n.context().cache().context().wal();
+     }
+
+    /**
+     * Disable/enable VAL.
+     *
+     * @param n Node.
+     * @param disable Disable flag.
+     * @throws Exception If failed.
+     */
+    protected static void disableWal(IgniteEx n, boolean disable) throws Exception {
+        WALDisableContext walDisableCtx = n.context().cache().context().walState().walDisableContext();
+
+        setFieldValue(walDisableCtx, "disableWal", disable);
+
+        assertEquals(disable, walDisableCtx.check());
+
+        WALPointer logPtr = walMgr(n).log(new DataRecord(emptyList()));
+
+        if (disable)
+            assertNull(logPtr);
+        else
+            assertNotNull(logPtr);
+    }
 }