You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/11/01 09:55:46 UTC
[16/19] ignite git commit: ignite-6788 Ignite WAL reader fails on Tx
marker record for persistent store with new style folder naming
ignite-6788 Ignite WAL reader fails on Tx marker record for persistent store with new style folder naming
Signed-off-by: agura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5542c70d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5542c70d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5542c70d
Branch: refs/heads/ignite-6778
Commit: 5542c70da913ff1006973e25f5112152674880db
Parents: 67ca745
Author: dpavlov <dp...@gridgain.com>
Authored: Wed Nov 1 03:17:00 2017 +0300
Committer: agura <ag...@apache.org>
Committed: Wed Nov 1 03:17:00 2017 +0300
----------------------------------------------------------------------
.../wal/reader/StandaloneGridKernalContext.java | 20 +-
.../db/wal/reader/IgniteWalReaderTest.java | 283 ++++++++++++-------
2 files changed, 195 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
index c0c3650..485458b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.cache.persistence.wal.reader;
import java.io.File;
-import java.io.Serializable;
import java.lang.reflect.Field;
import java.util.Iterator;
import java.util.List;
@@ -49,8 +48,8 @@ import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager;
import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
-import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings;
+import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver;
import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
@@ -95,6 +94,9 @@ import org.jetbrains.annotations.Nullable;
* Dummy grid kernal context
*/
public class StandaloneGridKernalContext implements GridKernalContext {
+ /** Binary metadata file store folderŅ */
+ public static final String BINARY_META_FOLDER = "binary_meta";
+
/** Config for fake Ignite instance. */
private final IgniteConfiguration cfg;
@@ -123,13 +125,12 @@ public class StandaloneGridKernalContext implements GridKernalContext {
* Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects will be provided <br>
*/
StandaloneGridKernalContext(IgniteLogger log,
- @Nullable final File binaryMetadataFileStoreDir,
- @Nullable final File marshallerMappingFileStoreDir) throws IgniteCheckedException {
+ @Nullable File binaryMetadataFileStoreDir,
+ @Nullable File marshallerMappingFileStoreDir) throws IgniteCheckedException {
this.log = log;
try {
- pluginProc = new StandaloneIgnitePluginProcessor(
- this, config());
+ pluginProc = new StandaloneIgnitePluginProcessor(this, config());
}
catch (IgniteCheckedException e) {
throw new IllegalStateException("Must not fail on empty providers list.", e);
@@ -137,7 +138,12 @@ public class StandaloneGridKernalContext implements GridKernalContext {
this.marshallerCtx = new MarshallerContextImpl(null);
this.cfg = prepareIgniteConfiguration();
- this.cacheObjProcessor = binaryMetadataFileStoreDir != null ? binaryProcessor(this, binaryMetadataFileStoreDir) : null;
+
+ // Fake folder provided to perform processor startup on empty folder.
+ if (binaryMetadataFileStoreDir == null)
+ binaryMetadataFileStoreDir = new File(BINARY_META_FOLDER).getAbsoluteFile();
+
+ this.cacheObjProcessor = binaryProcessor(this, binaryMetadataFileStoreDir);
if (marshallerMappingFileStoreDir != null) {
marshallerCtx.setMarshallerMappingFileStoreDir(marshallerMappingFileStoreDir);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
index 88eacef..9348a68 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java
@@ -31,6 +31,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -40,6 +41,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteEvents;
+import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.binary.BinaryObject;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheRebalanceMode;
@@ -68,8 +70,12 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.reader.Ignite
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.logger.NullLogger;
+import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.jetbrains.annotations.NotNull;
@@ -77,6 +83,7 @@ import org.jetbrains.annotations.Nullable;
import org.junit.Assert;
import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED;
+import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE;
import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor.genNewStyleSubfolderName;
@@ -94,12 +101,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
/** additional cache for testing different combinations of types in WAL */
private static final String CACHE_ADDL_NAME = "cache1";
- /** Delete DB dir before test. */
- private static final boolean deleteBefore = true;
-
- /** Delete DB dir after test. */
- private static final boolean deleteAfter = true;
-
/** Dump records to logger. Should be false for non local run */
private static final boolean dumpRecords = false;
@@ -115,6 +116,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
/** Custom wal mode. */
private WALMode customWalMode;
+ /** Clear properties in afterTest method() */
+ private boolean clearProperties;
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
final IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -151,23 +155,22 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
@Override protected void beforeTest() throws Exception {
stopAllGrids();
- if (deleteBefore)
- deleteWorkFiles();
+ deleteWorkFiles();
}
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
stopAllGrids();
- if (deleteAfter)
- deleteWorkFiles();
+ if (clearProperties)
+ System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS);
}
/**
* @throws IgniteCheckedException If failed.
*/
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false));
+ private void deleteWorkFiles() throws Exception {
+ GridTestUtils.deleteDbFiles();
}
/**
@@ -197,17 +200,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final int cntUsingMockIter = iterateAndCount(it, false);
log.info("Total records loaded " + cntUsingMockIter);
- assert cntUsingMockIter > 0;
- assert cntUsingMockIter > cacheObjectsToWrite;
+ assertTrue(cntUsingMockIter > 0);
+ assertTrue(cntUsingMockIter > cacheObjectsToWrite);
final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName);
final File walWorkDirWithConsistentId = new File(wal, subfolderName);
-
- final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
- final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
- final File marshaller = U.resolveWorkDirectory(workDir, "marshaller", false);
-
- final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithConsId, marshaller);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId));
log.info("Total records loaded using directory : " + cntArchiveDir);
@@ -218,12 +216,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile);
- assert cntArchiveFileByFile > cacheObjectsToWrite;
- assert cntArchiveDir > cacheObjectsToWrite;
- assert cntArchiveDir == cntArchiveFileByFile;
+ assertTrue(cntArchiveFileByFile > cacheObjectsToWrite);
+ assertTrue(cntArchiveDir > cacheObjectsToWrite);
+ assertTrue(cntArchiveDir == cntArchiveFileByFile);
//really count2 may be less because work dir correct loading is not supported yet
- assert cntUsingMockIter >= cntArchiveDir
- : "Mock based reader loaded " + cntUsingMockIter + " records but standalone has loaded only " + cntArchiveDir;
+ assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " +
+ "but standalone has loaded only " + cntArchiveDir,
+ cntUsingMockIter >= cntArchiveDir);
final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER);
@@ -231,10 +230,10 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
log.info("Total records loaded from work: " + cntWork);
- assert cntWork + cntArchiveFileByFile == cntUsingMockIter
- : "Work iterator loaded [" + cntWork + "] " +
- "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
- "mock iterator [" + cntUsingMockIter + "]";
+ assertTrue("Work iterator loaded [" + cntWork + "] " +
+ "Archive iterator loaded [" + cntArchiveFileByFile + "]; " +
+ "mock iterator [" + cntUsingMockIter + "]",
+ cntWork + cntArchiveFileByFile == cntUsingMockIter);
}
/**
@@ -312,7 +311,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
putDummyRecords(ignite, 500);
stopGrid("node0");
- assert evtRecorded.get();
+ assertTrue(evtRecorded.get());
}
/**
@@ -407,7 +406,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
archiveSegmentForInactivity.await(archiveIncompleteSegmentAfterInactivityMs + 1001, TimeUnit.MILLISECONDS);
stopGrid("node0");
- assert recordedAfterSleep;
+ assertTrue(recordedAfterSleep);
}
/**
@@ -460,8 +459,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
binaryMetaWithConsId,
marshallerMapping);
- final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>() {
- @Override public void accept(Object key, Object val) {
+ final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {
+ @Override public void apply(Object key, Object val) {
boolean rmv = remove(ctrlMap, key, val);
if (!rmv)
log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]");
@@ -471,14 +470,14 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
assertEquals(indexedObj.iVal, indexedObj.jVal);
assertEquals(indexedObj.iVal, key);
for (byte datum : indexedObj.getData()) {
- assert datum >= 'A' && datum <= 'A' + 10;
+ assertTrue(datum >= 'A' && datum <= 'A' + 10);
}
}
}
};
scanIterateAndCount(factory, workDir, subfolderName, cntEntries, txCnt, objConsumer, null);
- assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries " + ctrlMap;
+ assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty());
}
/**
@@ -510,8 +509,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final String subfolderName,
final int minCntEntries,
final int minTxCnt,
- @Nullable final BiConsumer<Object, Object> objConsumer,
- @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException {
+ @Nullable final IgniteBiInClosure<Object, Object> objConsumer,
+ @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false);
final File wal = new File(db, "wal");
@@ -544,8 +543,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final int entriesWork = valuesSum(cntWork.values());
log.info("Archive directory: Tx found " + txCntObservedWork + " entries " + entriesWork);
- assert entriesArch + entriesWork >= minCntEntries;
- assert txCntObservedWork + txCntObservedArch >= minTxCnt;
+ assertTrue("entriesArch=" + entriesArch + " + entriesWork=" + entriesWork
+ + " >= minCntEntries=" + minCntEntries,
+ entriesArch + entriesWork >= minCntEntries);
+ assertTrue("txCntObservedWork=" + txCntObservedWork + " + txCntObservedArch=" + txCntObservedArch
+ + " >= minTxCnt=" + minTxCnt,
+ txCntObservedWork + txCntObservedArch >= minTxCnt);
}
/**
@@ -611,11 +614,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final File binaryMetaWithNodeSubfolder = new File(binaryMeta, subfolderName);
final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false);
- final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
- binaryMetaWithNodeSubfolder,
- marshallerMapping);
- final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>() {
- @Override public void accept(Object key, Object val) {
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+ final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() {
+ @Override public void apply(Object key, Object val) {
log.info("K: [" + key + ", " +
(key != null ? key.getClass().getName() : "?") + "]" +
" V: [" + val + ", " +
@@ -625,12 +626,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
String msg = "Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]";
log.error(msg);
}
- assert !(val instanceof BinaryObject);
+ assertFalse(val instanceof BinaryObject);
}
};
- final Consumer<DataRecord> toStrChecker = new Consumer<DataRecord>() {
- @Override public void accept(DataRecord record) {
+ final IgniteInClosure<DataRecord> toStrChecker = new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord record) {
String strRepresentation = record.toString();
for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) {
final String next = iter.next();
@@ -643,17 +644,17 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
};
scanIterateAndCount(factory, workDir, subfolderName, cntEntries, 0, objConsumer, toStrChecker);
- assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries: " + ctrlMap;
- assert ctrlStringsToSearch.isEmpty() : " Control Map for strings in entries is not empty after" +
- " reading records: " + ctrlStringsToSearch;
+ assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty());
+ assertTrue(" Control Map for strings in entries is not empty after" +
+ " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty());
//Validate same WAL log with flag binary objects only
final IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log, PAGE_SIZE,
binaryMetaWithNodeSubfolder,
marshallerMapping,
true);
- final BiConsumer<Object, Object> binObjConsumer = new BiConsumer<Object, Object>() {
- @Override public void accept(Object key, Object val) {
+ final IgniteBiInClosure<Object, Object> binObjConsumer = new IgniteBiInClosure<Object, Object>() {
+ @Override public void apply(Object key, Object val) {
log.info("K(KeepBinary): [" + key + ", " +
(key != null ? key.getClass().getName() : "?") + "]" +
" V(KeepBinary): [" + val + ", " +
@@ -693,15 +694,15 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
byte data[] = binaryObj.field("data");
for (byte datum : data) {
- assert datum >= 'A' && datum <= 'A' + 10;
+ assertTrue(datum >= 'A' && datum <= 'A' + 10);
}
}
}
}
};
- final Consumer<DataRecord> binObjToStrChecker = new Consumer<DataRecord>() {
- @Override public void accept(DataRecord record) {
+ final IgniteInClosure<DataRecord> binObjToStrChecker = new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord record) {
String strRepresentation = record.toString();
for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) {
final String next = iter.next();
@@ -714,9 +715,11 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
};
scanIterateAndCount(keepBinFactory, workDir, subfolderName, cntEntries, 0, binObjConsumer, binObjToStrChecker);
- assert ctrlMapForBinaryObjects.isEmpty() : " Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects;
- assert ctrlStringsForBinaryObjSearch.isEmpty() : " Control Map for strings in entries is not empty after" +
- " reading records: " + ctrlStringsForBinaryObjSearch;
+ assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects,
+ ctrlMapForBinaryObjects.isEmpty());
+ assertTrue(" Control Map for strings in entries is not empty after" +
+ " reading records: " + ctrlStringsForBinaryObjSearch,
+ ctrlStringsForBinaryObjSearch.isEmpty());
}
@@ -764,7 +767,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
stopGrid("node0");
final String workDir = U.defaultWorkDirectory();
- final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
scanIterateAndCount(factory, workDir, subfolderName, totalEntries, 0, null, null);
}
@@ -787,7 +790,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
stopGrid("node0");
final String workDir = U.defaultWorkDirectory();
- final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, null);
}
@@ -859,13 +862,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
stopGrid("node0");
final String workDir = U.defaultWorkDirectory();
- final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir);
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
final StringBuilder builder = new StringBuilder();
final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
- scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new Consumer<DataRecord>() {
- @Override public void accept(DataRecord dataRecord) {
+ scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord dataRecord) {
final List<DataEntry> entries = dataRecord.writeEntries();
builder.append("{");
@@ -902,14 +905,121 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
}
/**
- * @param subfolderName Subfolder name.
+ * Tests transaction generation and WAL for putAll cache operation.
+ * @throws Exception if failed.
+ */
+ public void testPutAllTxIntoTwoNodes() throws Exception {
+ final Ignite ignite = startGrid("node0");
+ final Ignite ignite1 = startGrid(1);
+
+ ignite.active(true);
+
+ final Map<Object, IndexedObject> map = new TreeMap<>();
+
+ final int cntEntries = 1000;
+ for (int i = 0; i < cntEntries; i++)
+ map.put(i, new IndexedObject(i));
+
+ ignite.cache(CACHE_NAME).putAll(map);
+
+ ignite.active(false);
+
+ final String subfolderName = genDbSubfolderName(ignite, 0);
+ final String subfolderName1 = genDbSubfolderName(ignite1, 1);
+
+ stopAllGrids();
+
+ final String workDir = U.defaultWorkDirectory();
+ final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName);
+
+ final StringBuilder builder = new StringBuilder();
+ final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class);
+
+ final IgniteInClosure<DataRecord> drHnd = new IgniteInClosure<DataRecord>() {
+ @Override public void apply(DataRecord dataRecord) {
+ final List<DataEntry> entries = dataRecord.writeEntries();
+
+ builder.append("{");
+ for (DataEntry entry : entries) {
+ final GridCacheOperation op = entry.op();
+ final Integer cnt = operationsFound.get(op);
+
+ operationsFound.put(op, cnt == null ? 1 : (cnt + 1));
+
+ if (entry instanceof UnwrapDataEntry) {
+ final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry;
+
+ builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey());
+ final GridCacheVersion ver = entry.nearXidVersion();
+
+ builder.append(", ");
+
+ if (ver != null)
+ builder.append("tx=").append(ver).append(", ");
+ }
+ }
+
+ builder.append("}\n");
+ }
+ };
+ scanIterateAndCount(factory, workDir, subfolderName, 1, 1, null, drHnd);
+ scanIterateAndCount(factory, workDir, subfolderName1, 1, 1, null, drHnd);
+
+ final Integer createsFound = operationsFound.get(CREATE);
+
+ if (log.isInfoEnabled())
+ log.info(builder.toString());
+
+ assertTrue("Create operations should be found in log: " + operationsFound,
+ createsFound != null && createsFound > 0);
+
+ assertTrue("Create operations count should be at least " + cntEntries + " in log: " + operationsFound,
+ createsFound != null && createsFound >= cntEntries);
+
+ }
+
+ /**
+ * Tests transaction generation and WAL for putAll cache operation.
+ * @throws Exception if failed.
+ */
+ public void testTxRecordsReadWoBinaryMeta() throws Exception {
+ clearProperties = true;
+ System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true");
+
+ final Ignite ignite = startGrid("node0");
+ ignite.active(true);
+
+ final Map<Object, IndexedObject> map = new TreeMap<>();
+
+ for (int i = 0; i < 1000; i++)
+ map.put(i, new IndexedObject(i));
+
+ ignite.cache(CACHE_NAME).putAll(map);
+
+ ignite.active(false);
+
+ final String workDir = U.defaultWorkDirectory();
+ final String subfolderName = genDbSubfolderName(ignite, 0);
+ stopAllGrids();
+
+ IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(),
+ PAGE_SIZE,
+ null,
+ null,
+ false);
+
+ scanIterateAndCount(factory, workDir, subfolderName, 1000, 1, null, null);
+ }
+
+ /**
* @param workDir Work directory.
+ * @param subfolderName Subfolder name.
* @return WAL iterator factory.
* @throws IgniteCheckedException If failed.
*/
@NotNull private IgniteWalIteratorFactory createWalIteratorFactory(
- String subfolderName,
- String workDir
+ final String workDir,
+ final String subfolderName
) throws IgniteCheckedException {
final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false);
final File binaryMetaWithConsId = new File(binaryMeta, subfolderName);
@@ -918,7 +1028,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
return new IgniteWalIteratorFactory(log,
PAGE_SIZE,
binaryMetaWithConsId,
- marshallerMapping);
+ marshallerMapping,
+ false);
}
/**
@@ -943,8 +1054,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
*/
private Map<GridCacheVersion, Integer> iterateAndCountDataRecord(
final WALIterator walIter,
- @Nullable final BiConsumer<Object, Object> cacheObjHnd,
- @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException {
+ @Nullable final IgniteBiInClosure<Object, Object> cacheObjHnd,
+ @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException {
final Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>();
@@ -957,7 +1068,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
final DataRecord dataRecord = (DataRecord)walRecord;
if (dataRecordHnd != null)
- dataRecordHnd.accept(dataRecord);
+ dataRecordHnd.apply(dataRecord);
final List<DataEntry> entries = dataRecord.writeEntries();
for (DataEntry entry : entries) {
@@ -990,7 +1101,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
"; Value: " + unwrappedValObj);
if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null))
- cacheObjHnd.accept(unwrappedKeyObj, unwrappedValObj);
+ cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj);
final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId);
entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 : entriesUnderTx + 1);
@@ -1008,36 +1119,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest {
return entriesUnderTxFound;
}
- /**
- * Represents an operation that accepts a single input argument and returns no
- * result.
- *
- * @param <T>
- */
- private interface Consumer<T> {
- /**
- * Performs this operation on the given argument.
- *
- * @param t the input argument
- */
- public void accept(T t);
- }
-
- /**
- * Represents an operation that accepts two input arguments and returns no
- * result.
- *
- * @param <T>
- */
- private interface BiConsumer<T, U> {
- /**
- * Performs this operation on the given argument.
- *
- * @param t the input argument
- */
- public void accept(T t, U u);
- }
-
/** Enum for cover binaryObject enum save/load */
enum TestEnum {
/** */A, /** */B, /** */C