You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2021/03/03 11:08:48 UTC
[ignite] branch master updated: IGNITE-14246 Parameter to filter
certain pages from WAL when reading with WAL converter - Fixes #8834.
This is an automated email from the ASF dual-hosted git repository.
sergeychugunov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 235a779 IGNITE-14246 Parameter to filter certain pages from WAL when reading with WAL converter - Fixes #8834.
235a779 is described below
commit 235a779dd11894b0870cff8687ad286748f7969a
Author: Kirill Tkalenko <tk...@yandex.ru>
AuthorDate: Wed Mar 3 13:58:07 2021 +0300
IGNITE-14246 Parameter to filter certain pages from WAL when reading with WAL converter - Fixes #8834.
Signed-off-by: Sergey Chugunov <se...@gmail.com>
---
.../development/utils/IgniteWalConverter.java | 61 ++++++-
.../utils/IgniteWalConverterArguments.java | 137 ++++++++++++++-
.../utils/IgniteWalConverterArgumentsTest.java | 191 +++++++++++++++++----
.../development/utils/IgniteWalConverterTest.java | 125 +++++++++++---
4 files changed, 442 insertions(+), 72 deletions(-)
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index e0d55c7..838ba9b 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -19,8 +19,15 @@ package org.apache.ignite.development.utils;
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
@@ -30,6 +37,7 @@ import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.reader.FilteredWalIterator;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
@@ -38,10 +46,15 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2InnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2LeafIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2MvccInnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2MvccLeafIO;
-import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.checkpoint;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.pageOwner;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.partitionMetaStateUpdate;
+
/**
* Print WAL log data in human-readable form.
*/
@@ -94,7 +107,7 @@ public class IgniteWalConverter {
boolean printAlways = F.isEmpty(params.getRecordTypes());
- try (WALIterator stIt = factory.iterator(iteratorParametersBuilder)) {
+ try (WALIterator stIt = walIterator(factory.iterator(iteratorParametersBuilder), params.getPages())) {
String currentWalPath = null;
while (stIt.hasNextX()) {
@@ -155,27 +168,29 @@ public class IgniteWalConverter {
}
/**
- * Get current wal file path, used in {@code WALIterator}
+ * Get current wal file path, used in {@code WALIterator}.
*
* @param it WALIterator.
* @return Current wal file path.
*/
private static String getCurrentWalFilePath(WALIterator it) {
- String result = null;
+ String res = null;
try {
- final Integer curIdx = IgniteUtils.field(it, "curIdx");
+ WALIterator walIter = it instanceof FilteredWalIterator ? U.field(it, "delegateWalIter") : it;
+
+ Integer curIdx = U.field(walIter, "curIdx");
- final List<FileDescriptor> walFileDescriptors = IgniteUtils.field(it, "walFileDescriptors");
+ List<FileDescriptor> walFileDescriptors = U.field(walIter, "walFileDescriptors");
- if (curIdx != null && walFileDescriptors != null && !walFileDescriptors.isEmpty())
- result = walFileDescriptors.get(curIdx).getAbsolutePath();
+ if (curIdx != null && walFileDescriptors != null && curIdx < walFileDescriptors.size())
+ res = walFileDescriptors.get(curIdx).getAbsolutePath();
}
catch (Exception e) {
e.printStackTrace();
}
- return result;
+ return res;
}
/**
@@ -201,4 +216,32 @@ public class IgniteWalConverter {
return walRecord.toString();
}
+
+ /**
+ * Getting WAL iterator.
+ *
+ * @param walIter WAL iterator.
+ * @param pageIds Pages for searching in format grpId:pageId.
+ * @return WAL iterator.
+ */
+ private static WALIterator walIterator(
+ WALIterator walIter,
+ Collection<T2<Integer, Long>> pageIds
+ ) throws IgniteCheckedException {
+ Predicate<IgniteBiTuple<WALPointer, WALRecord>> filter = null;
+
+ if (!pageIds.isEmpty()) {
+ Set<T2<Integer, Long>> grpAndPageIds0 = new HashSet<>(pageIds);
+
+ // Collect all (group, partition) partition pairs.
+ Set<T2<Integer, Integer>> grpAndParts = grpAndPageIds0.stream()
+ .map((tup) -> new T2<>(tup.get1(), PageIdUtils.partId(tup.get2())))
+ .collect(Collectors.toSet());
+
+ // Build WAL filter. (Checkoint, Page, Partition meta)
+ filter = checkpoint().or(pageOwner(grpAndPageIds0)).or(partitionMetaStateUpdate(grpAndParts));
+ }
+
+ return filter != null ? new FilteredWalIterator(walIter, filter) : walIter;
+ }
}
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java
index af3bfb8..3e368cc 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverterArguments.java
@@ -17,16 +17,27 @@
package org.apache.ignite.development.utils;
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Date;
import java.util.HashSet;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.jetbrains.annotations.Nullable;
+
+import static java.util.Collections.emptyList;
/**
* Parameters for IgniteWalConverter with parsed and validated.
@@ -71,6 +82,12 @@ public class IgniteWalConverterArguments {
/** */
private static final String SKIP_CRC = "skipCrc";
+ /** Argument "pages". */
+ private static final String PAGES = "pages";
+
+ /** Record pattern for {@link #PAGES}. */
+ private static final Pattern PAGE_ID_PATTERN = Pattern.compile("(\\d+):(\\d+)");
+
/** Path to dir with wal files. */
private final File walDir;
@@ -110,7 +127,12 @@ public class IgniteWalConverterArguments {
/** Skip CRC calculation/check flag */
private final boolean skipCrc;
+ /** Pages for searching in format grpId:pageId. */
+ private final Collection<T2<Integer, Long>> pages;
+
/**
+ * Constructor.
+ *
* @param walDir Path to dir with wal files.
* @param walArchiveDir Path to dir with archive wal files.
* @param pageSize Size of pages, which was selected for file store (1024, 2048, 4096, etc).
@@ -124,12 +146,13 @@ public class IgniteWalConverterArguments {
* @param processSensitiveData Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5).
* @param printStat Write summary statistics for WAL.
* @param skipCrc Skip CRC calculation/check flag.
+ * @param pages Pages for searching in format grpId:pageId.
*/
public IgniteWalConverterArguments(File walDir, File walArchiveDir, int pageSize,
File binaryMetadataFileStoreDir, File marshallerMappingFileStoreDir, boolean keepBinary,
Set<WALRecord.RecordType> recordTypes, Long fromTime, Long toTime, String recordContainsText,
ProcessSensitiveData processSensitiveData,
- boolean printStat, boolean skipCrc) {
+ boolean printStat, boolean skipCrc, Collection<T2<Integer, Long>> pages) {
this.walDir = walDir;
this.walArchiveDir = walArchiveDir;
this.pageSize = pageSize;
@@ -143,6 +166,7 @@ public class IgniteWalConverterArguments {
this.processSensitiveData = processSensitiveData;
this.printStat = printStat;
this.skipCrc = skipCrc;
+ this.pages = pages;
}
/**
@@ -263,12 +287,21 @@ public class IgniteWalConverterArguments {
}
/**
+ * Return pages for searching in format grpId:pageId.
+ *
+ * @return Pages.
+ */
+ public Collection<T2<Integer, Long>> getPages() {
+ return pages;
+ }
+
+ /**
* Parse command line arguments and return filled IgniteWalConverterArguments
*
* @param args Command line arguments.
* @return IgniteWalConverterArguments.
*/
- public static IgniteWalConverterArguments parse(final PrintStream out, String args[]) {
+ public static IgniteWalConverterArguments parse(final PrintStream out, String... args) {
if (args == null || args.length < 1) {
out.println("Print WAL log data in human-readable form.");
out.println("You need to provide:");
@@ -285,6 +318,8 @@ public class IgniteWalConverterArguments {
out.println(" processSensitiveData (Optional) Strategy for the processing of sensitive data (SHOW, HIDE, HASH, MD5). Default SHOW.");
out.println(" printStat Write summary statistics for WAL. Default false.");
out.println(" skipCrc Skip CRC calculation/check flag. Default false.");
+ out.println(" pages (Optional) Comma-separated pages or path to file with " +
+ "pages on each line in grpId:pageId format.");
out.println("For example:");
out.println(" walDir=/work/db/wal");
out.println(" walArchiveDir=/work/db/wal_archive");
@@ -298,6 +333,7 @@ public class IgniteWalConverterArguments {
out.println(" recordContainsText=search string");
out.println(" processSensitiveData=SHOW");
out.println(" skipCrc=true");
+ out.println(" pages=123456:789456123,123456:789456124");
return null;
}
@@ -314,6 +350,7 @@ public class IgniteWalConverterArguments {
ProcessSensitiveData processSensitiveData = ProcessSensitiveData.SHOW;
boolean printStat = false;
boolean skipCrc = false;
+ Collection<T2<Integer, Long>> pages = emptyList();
for (String arg : args) {
if (arg.startsWith(WAL_DIR + "=")) {
@@ -420,6 +457,13 @@ public class IgniteWalConverterArguments {
else if (arg.startsWith(SKIP_CRC + "=")) {
skipCrc = parseBoolean(SKIP_CRC, arg.substring(SKIP_CRC.length() + 1));
}
+ else if (arg.startsWith(PAGES + "=")) {
+ String pagesStr = arg.replace(PAGES + "=", "");
+
+ File pagesFile = new File(pagesStr);
+
+ pages = pagesFile.exists() ? parsePageIds(pagesFile) : parsePageIds(pagesStr.split(","));
+ }
}
if (walDir == null && walArchiveDir == null)
@@ -459,9 +503,13 @@ public class IgniteWalConverterArguments {
out.printf("\t%s = %b\n", SKIP_CRC, skipCrc);
+ if (!pages.isEmpty())
+ out.printf("\t%s = %s\n", PAGES, pages);
+
return new IgniteWalConverterArguments(walDir, walArchiveDir, pageSize,
binaryMetadataFileStoreDir, marshallerMappingFileStoreDir,
- keepBinary, recordTypes, fromTime, toTime, recordContainsText, processSensitiveData, printStat, skipCrc);
+ keepBinary, recordTypes, fromTime, toTime, recordContainsText, processSensitiveData, printStat, skipCrc,
+ pages);
}
/**
@@ -488,4 +536,87 @@ public class IgniteWalConverterArguments {
else
throw new IllegalArgumentException("Incorrect flag " + name + ", valid value: true or false. Error parse: " + value);
}
+
+ /**
+ * Parsing and checking the string representation of the page in grpId:pageId format.
+ * Example: 123:456.
+ *
+ * @param s String value.
+ * @return Parsed value.
+ * @throws IllegalArgumentException If the string value is invalid.
+ */
+ static T2<Integer, Long> parsePageId(@Nullable String s) throws IllegalArgumentException {
+ if (s == null)
+ throw new IllegalArgumentException("Null value.");
+ else if (s.isEmpty())
+ throw new IllegalArgumentException("Empty value.");
+
+ Matcher m = PAGE_ID_PATTERN.matcher(s);
+
+ if (!m.matches()) {
+ throw new IllegalArgumentException("Incorrect value " + s + ", valid format: grpId:pageId. " +
+ "Example: 123:456");
+ }
+
+ return new T2<>(Integer.parseInt(m.group(1)), Long.parseLong(m.group(2)));
+ }
+
+ /**
+ * Parsing a file in which each line is expected to be grpId:pageId format.
+ *
+ * @param f File.
+ * @return Parsed pages.
+ * @throws IllegalArgumentException If there is an error when working with a file or parsing lines.
+ * @see #parsePageId
+ */
+ static Collection<T2<Integer, Long>> parsePageIds(File f) throws IllegalArgumentException {
+ try (BufferedReader reader = new BufferedReader(new FileReader(f))) {
+ int i = 0;
+ String s;
+
+ Collection<T2<Integer, Long>> res = new ArrayList<>();
+
+ while ((s = reader.readLine()) != null) {
+ try {
+ res.add(parsePageId(s));
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "Error parsing value \"" + s + "\" on " + i + " line of the file: " + f.getAbsolutePath(),
+ e
+ );
+ }
+
+ i++;
+ }
+
+ return res.isEmpty() ? emptyList() : res;
+ }
+ catch (IOException e) {
+ throw new IllegalArgumentException("Error when working with the file: " + f.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * Parsing strings in which each element is expected to be in grpId:pageId format.
+ *
+ * @param strs String values.
+ * @return Parsed pages.
+ * @throws IllegalArgumentException If there is an error parsing the strs.
+ * @see #parsePageId
+ */
+ static Collection<T2<Integer, Long>> parsePageIds(String... strs) throws IllegalArgumentException {
+ Collection<T2<Integer, Long>> res = new ArrayList<>();
+
+ for (int i = 0; i < strs.length; i++) {
+ try {
+ res.add(parsePageId(strs[i]));
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Error parsing value \"" + strs[i] + "\" of " + i + " element", e);
+ }
+ }
+
+ return res.isEmpty() ? emptyList() : res;
+ }
}
diff --git a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java
index 84098fb..b5c6976 100644
--- a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java
+++ b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterArgumentsTest.java
@@ -23,14 +23,21 @@ import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
-
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.junit.Assert;
import org.junit.Test;
+import static java.nio.charset.Charset.defaultCharset;
+import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parse;
+import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parsePageId;
+import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parsePageIds;
+import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
+
/**
* Test for IgniteWalConverterArguments
*/
@@ -54,7 +61,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
public void testViewHelp() throws Exception {
final ByteArrayOutputStream out = new ByteArrayOutputStream();
- final IgniteWalConverterArguments parseArgs = IgniteWalConverterArguments.parse(new PrintStream(out), null);
+ final IgniteWalConverterArguments parseArgs = parse(new PrintStream(out), null);
Assert.assertNull(parseArgs);
@@ -82,8 +89,8 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testRequiredWalDir() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
- IgniteWalConverterArguments.parse(System.out, new String[] {"pageSize=4096"});
+ assertThrows(log, () -> {
+ parse(System.out, new String[] {"pageSize=4096"});
}, IgniteException.class, "The paths to the WAL files are not specified.");
}
@@ -94,8 +101,8 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectWalDir() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
- IgniteWalConverterArguments.parse(System.out, new String[] {"walDir=non_existing_path"});
+ assertThrows(log, () -> {
+ parse(System.out, new String[] {"walDir=non_existing_path"});
}, IgniteException.class, "Incorrect path to dir with wal files: non_existing_path");
}
@@ -106,8 +113,8 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectWalArchiveDir() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
- IgniteWalConverterArguments.parse(System.out, new String[] {"walArchiveDir=non_existing_path"});
+ assertThrows(log, () -> {
+ parse(System.out, new String[] {"walArchiveDir=non_existing_path"});
}, IgniteException.class, "Incorrect path to dir with archive wal files: non_existing_path");
}
@@ -118,7 +125,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectPageSize() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -127,7 +134,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"pageSize=not_integer"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect page size. Error parse: not_integer");
}
@@ -138,7 +145,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectBinaryMetadataFileStoreDir() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -147,7 +154,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"binaryMetadataFileStoreDir=non_existing_path"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect path to dir with binary meta files: non_existing_path");
}
@@ -158,7 +165,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectMarshallerMappingFileStoreDir() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -167,7 +174,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"marshallerMappingFileStoreDir=non_existing_path"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect path to dir with marshaller files: non_existing_path");
}
@@ -178,7 +185,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectKeepBinary() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -187,7 +194,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"keepBinary=not_boolean"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect flag keepBinary, valid value: true or false. Error parse: not_boolean");
}
@@ -198,7 +205,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectRecordTypes() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -207,7 +214,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"recordTypes=not_exist"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Unknown record types: [not_exist].");
}
@@ -218,7 +225,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectSeveralRecordTypes() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -227,7 +234,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"recordTypes=not_exist1,not_exist2"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Unknown record types: [not_exist1, not_exist2].");
}
@@ -238,7 +245,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectWalTimeFromMillis() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -247,7 +254,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"walTimeFromMillis=not_long"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect walTimeFromMillis. Error parse: not_long");
}
@@ -258,7 +265,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectWalTimeToMillis() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -267,7 +274,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"walTimeToMillis=not_long"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect walTimeToMillis. Error parse: not_long");
}
@@ -278,7 +285,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectProcessSensitiveData() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -287,7 +294,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"processSensitiveData=unknown"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Unknown processSensitiveData: unknown. Supported: ");
}
@@ -298,7 +305,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectPrintStat() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -307,7 +314,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"printStat=not_boolean"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect flag printStat, valid value: true or false. Error parse: not_boolean");
}
@@ -318,7 +325,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
*/
@Test
public void testIncorrectSkipCrc() throws Exception {
- GridTestUtils.assertThrows(log, () -> {
+ assertThrows(log, () -> {
final File wal = File.createTempFile("wal", "");
wal.deleteOnExit();
@@ -327,7 +334,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"skipCrc=not_boolean"
};
- IgniteWalConverterArguments.parse(System.out, args);
+ parse(System.out, args);
}, IgniteException.class, "Incorrect flag skipCrc, valid value: true or false. Error parse: not_boolean");
}
@@ -345,7 +352,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"walDir=" + wal.getAbsolutePath()
};
- final IgniteWalConverterArguments parseArgs = IgniteWalConverterArguments.parse(System.out, args);
+ final IgniteWalConverterArguments parseArgs = parse(System.out, args);
Assert.assertEquals(4096, parseArgs.getPageSize());
Assert.assertNull(parseArgs.getBinaryMetadataFileStoreDir());
@@ -392,7 +399,7 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
"printStat=true",
"skipCrc=true"};
- final IgniteWalConverterArguments parseArgs = IgniteWalConverterArguments.parse(System.out, args);
+ final IgniteWalConverterArguments parseArgs = parse(System.out, args);
Assert.assertEquals(wal, parseArgs.getWalDir());
Assert.assertEquals(walArchive, parseArgs.getWalArchiveDir());
Assert.assertEquals(2048, parseArgs.getPageSize());
@@ -408,4 +415,122 @@ public class IgniteWalConverterArgumentsTest extends GridCommonAbstractTest {
Assert.assertTrue(parseArgs.isPrintStat());
Assert.assertTrue(parseArgs.isSkipCrc());
}
+
+ /**
+ * Checking the correctness of the method {@link IgniteWalConverterArguments#parsePageId}.
+ */
+ @Test
+ public void testParsePageId() {
+ String[] invalidValues = {
+ null,
+ "",
+ " ",
+ "a",
+ "a:",
+ "a:b",
+ "a:b",
+ "a:1",
+ "1:b",
+ "1;1",
+ "1a:1",
+ "1:1b",
+ "1:1:1",
+ };
+
+ for (String v : invalidValues)
+ assertThrows(log, () -> parsePageId(v), IllegalArgumentException.class, null);
+
+ assertEquals(new T2<>(1, 1L), parsePageId("1:1"));
+ }
+
+ /**
+ * Checking the correctness of the method {@link IgniteWalConverterArguments#parsePageIds(File)}.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testParsePageIdsFile() throws Exception {
+ File f = new File(System.getProperty("java.io.tmpdir"), "test");
+
+ try {
+ assertThrows(log, () -> parsePageIds(f), IllegalArgumentException.class, null);
+
+ assertTrue(f.createNewFile());
+
+ assertTrue(parsePageIds(f).isEmpty());
+
+ U.writeStringToFile(f, "a:b", defaultCharset().toString(), false);
+ assertThrows(log, () -> parsePageIds(f), IllegalArgumentException.class, null);
+
+ U.writeStringToFile(f, "1:1,1:1", defaultCharset().toString(), false);
+ assertThrows(log, () -> parsePageIds(f), IllegalArgumentException.class, null);
+
+ U.writeStringToFile(f, "1:1", defaultCharset().toString(), false);
+ assertEqualsCollections(F.asList(new T2<>(1, 1L)), parsePageIds(f));
+
+ U.writeStringToFile(f, U.nl() + "2:2", defaultCharset().toString(), true);
+ assertEqualsCollections(F.asList(new T2<>(1, 1L), new T2<>(2, 2L)), parsePageIds(f));
+ }
+ finally {
+ assertTrue(U.delete(f));
+ }
+ }
+
+ /**
+ * Checking the correctness of the method {@link IgniteWalConverterArguments#parsePageIds(String...)}.
+ */
+ @Test
+ public void testParsePageIdsStrings() {
+ assertTrue(parsePageIds().isEmpty());
+
+ assertThrows(log, () -> parsePageIds("a:b"), IllegalArgumentException.class, null);
+ assertThrows(log, () -> parsePageIds("1:1", "a:b"), IllegalArgumentException.class, null);
+
+ assertEqualsCollections(F.asList(new T2<>(1, 1L)), parsePageIds("1:1"));
+ assertEqualsCollections(F.asList(new T2<>(1, 1L), new T2<>(2, 2L)), parsePageIds("1:1", "2:2"));
+ }
+
+ /**
+ * Checking the correctness of parsing the argument "pages".
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testParsePagesArgument() throws IOException {
+ File walDir = new File(System.getProperty("java.io.tmpdir"), "walDir");
+
+ try {
+ assertTrue(walDir.mkdir());
+
+ String walDirStr = "walDir=" + walDir.getAbsolutePath();
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+
+ assertThrows(log, () -> parse(ps, walDirStr, "pages=1"), IllegalArgumentException.class, null);
+ assertThrows(log, () -> parse(ps, walDirStr, "pages="), IllegalArgumentException.class, null);
+
+ assertEqualsCollections(F.asList(new T2<>(1, 1L)), parse(ps, walDirStr, "pages=1:1").getPages());
+
+ File f = new File(System.getProperty("java.io.tmpdir"), "test");
+
+ try {
+ String pagesFileStr = "pages=" + f.getAbsolutePath();
+
+ assertThrows(log, () -> parse(ps, walDirStr, pagesFileStr), IllegalArgumentException.class, null);
+
+ assertTrue(f.createNewFile());
+ assertTrue(parse(ps, walDirStr, pagesFileStr).getPages().isEmpty());
+
+ U.writeStringToFile(f, "1:1", defaultCharset().toString(), false);
+ assertEqualsCollections(F.asList(new T2<>(1, 1L)), parse(ps, walDirStr, pagesFileStr).getPages());
+ }
+ finally {
+ assertTrue(U.delete(f));
+ }
+ }
+ finally {
+ assertTrue(U.delete(walDir));
+ }
+ }
}
diff --git a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java
index 1e38dbd..f3627b7 100644
--- a/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java
+++ b/modules/dev-utils/src/test/java/org/apache/ignite/development/utils/IgniteWalConverterTest.java
@@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.io.RandomAccessFile;
+import java.util.ArrayList;
import java.util.Base64;
import java.util.LinkedList;
import java.util.List;
@@ -34,12 +35,27 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.WALMode;
import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.WALPointer;
import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
+import org.apache.ignite.internal.util.lang.IgniteThrowableConsumer;
+import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
import org.junit.Test;
+import static java.util.Collections.emptyList;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_BINARY_METADATA_PATH;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_MARSHALLER_PATH;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH;
+import static org.apache.ignite.development.utils.IgniteWalConverter.convert;
+import static org.apache.ignite.development.utils.IgniteWalConverterArguments.parse;
+import static org.apache.ignite.testframework.GridTestUtils.assertContains;
+
/**
* Test for IgniteWalConverter
*/
@@ -65,6 +81,7 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
@Override protected void beforeTest() throws Exception {
super.beforeTest();
+ stopAllGrids();
cleanPersistenceDir();
}
@@ -145,24 +162,24 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
public void testIgniteWalConverter() throws Exception {
final List<Person> list = new LinkedList<>();
- final String nodeFolder = createWal(list);
+ final String nodeFolder = createWal(list, null);
final ByteArrayOutputStream outByte = new ByteArrayOutputStream();
final PrintStream out = new PrintStream(outByte);
final IgniteWalConverterArguments arg = new IgniteWalConverterArguments(
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false),
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false),
DataStorageConfiguration.DFLT_PAGE_SIZE,
- new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder),
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false),
+ new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false),
false,
null,
- null, null, null, null, true,true
+ null, null, null, null, true,true, emptyList()
);
- IgniteWalConverter.convert(out, arg);
+ convert(out, arg);
final String result = outByte.toString();
@@ -208,24 +225,24 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
public void testIgniteWalConverterWithOutBinaryMeta() throws Exception {
final List<Person> list = new LinkedList<>();
- createWal(list);
+ createWal(list, null);
final ByteArrayOutputStream outByte = new ByteArrayOutputStream();
final PrintStream out = new PrintStream(outByte);
final IgniteWalConverterArguments arg = new IgniteWalConverterArguments(
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false),
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false),
DataStorageConfiguration.DFLT_PAGE_SIZE,
null,
null,
false,
null,
- null, null, null, null, true,true
+ null, null, null, null, true,true, emptyList()
);
- IgniteWalConverter.convert(out, arg);
+ convert(out, arg);
final String result = outByte.toString();
@@ -274,9 +291,9 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
public void testIgniteWalConverterWithBrokenWal() throws Exception {
final List<Person> list = new LinkedList<>();
- final String nodeFolder = createWal(list);
+ final String nodeFolder = createWal(list, null);
- final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false);
+ final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false);
final File wal = new File(walDir, nodeFolder + File.separator + "0000000000000000.wal");
@@ -332,16 +349,16 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
final IgniteWalConverterArguments arg = new IgniteWalConverterArguments(
walDir,
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false),
DataStorageConfiguration.DFLT_PAGE_SIZE,
- new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder),
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false),
+ new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false),
false,
null,
- null, null, null, null, true,true
+ null, null, null, null, true,true, emptyList()
);
- IgniteWalConverter.convert(out, arg);
+ convert(out, arg);
final String result = outByte.toString();
@@ -392,9 +409,9 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
public void testIgniteWalConverterWithUnreadableWal() throws Exception {
final List<Person> list = new LinkedList<>();
- final String nodeFolder = createWal(list);
+ final String nodeFolder = createWal(list, null);
- final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_PATH, false);
+ final File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false);
final File wal = new File(walDir, nodeFolder + File.separator + "0000000000000000.wal");
@@ -436,16 +453,16 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
final IgniteWalConverterArguments arg = new IgniteWalConverterArguments(
walDir,
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_WAL_ARCHIVE_PATH, false),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_ARCHIVE_PATH, false),
DataStorageConfiguration.DFLT_PAGE_SIZE,
- new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_BINARY_METADATA_PATH, false), nodeFolder),
- U.resolveWorkDirectory(U.defaultWorkDirectory(), DataStorageConfiguration.DFLT_MARSHALLER_PATH, false),
+ new File(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_BINARY_METADATA_PATH, false), nodeFolder),
+ U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_MARSHALLER_PATH, false),
false,
null,
- null, null, null, null, true,true
+ null, null, null, null, true,true, emptyList()
);
- IgniteWalConverter.convert(out, arg);
+ convert(out, arg);
final String result = outByte.toString();
@@ -488,6 +505,53 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
}
/**
+ * Check that when using the "pages" argument we will see WalRecord with this pages in the utility output.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testPages() throws Exception {
+ List<T2<PageSnapshot, String>> walRecords = new ArrayList<>();
+
+ String nodeDir = createWal(new ArrayList<>(), n -> {
+ try (WALIterator walIter = n.context().cache().context().wal().replay(new WALPointer(0, 0, 0))) {
+ while (walIter.hasNextX()) {
+ WALRecord walRecord = walIter.nextX().get2();
+
+ if (walRecord instanceof PageSnapshot)
+ walRecords.add(new T2<>((PageSnapshot)walRecord, walRecord.toString()));
+ }
+ }
+ });
+
+ assertFalse(walRecords.isEmpty());
+
+ File walDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_WAL_PATH, false);
+ assertTrue(U.fileCount(walDir.toPath()) > 0);
+
+ File walNodeDir = new File(walDir, nodeDir);
+ assertTrue(U.fileCount(walNodeDir.toPath()) > 0);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream ps = new PrintStream(baos);
+
+ T2<PageSnapshot, String> expRec = walRecords.get(0);
+
+ IgniteWalConverterArguments args = parse(
+ ps,
+ "walDir=" + walDir.getAbsolutePath(),
+ "pages=" + expRec.get1().fullPageId().groupId() + ':' + expRec.get1().fullPageId().pageId(),
+ "skipCrc=" + true
+ );
+
+ baos.reset();
+
+ convert(ps, args);
+
+ assertContains(log, baos.toString(), expRec.get2());
+ }
+
+ /**
* Common part
* <ul>
* <li>Start node</li>
@@ -496,10 +560,14 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
* </ul>
*
* @param list Returns entities that have been added.
+ * @param afterPopulateConsumer
* @return Node folder name.
* @throws Exception
*/
- private String createWal(List<Person> list) throws Exception {
+ private String createWal(
+ List<Person> list,
+ @Nullable IgniteThrowableConsumer<IgniteEx> afterPopulateConsumer
+ ) throws Exception {
String nodeFolder;
try (final IgniteEx node = startGrid(0)) {
@@ -523,6 +591,9 @@ public class IgniteWalConverterTest extends GridCommonAbstractTest {
list.add(value);
}
+
+ if (afterPopulateConsumer != null)
+ afterPopulateConsumer.accept(node);
}
return nodeFolder;