You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by am...@apache.org on 2019/06/05 11:08:27 UTC
[ignite] 24/31: GG-18670 [IGNITE-11749] Implement automatic pages
history dump on CorruptedTreeException.
This is an automated email from the ASF dual-hosted git repository.
amashenkov pushed a commit to branch gg-19225
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 7de25e5256e9026c12d051c5f06cff7d91e56c35
Author: Anton Kalashnikov <ka...@yandex.ru>
AuthorDate: Mon Jun 3 17:24:48 2019 +0300
GG-18670 [IGNITE-11749] Implement automatic pages history dump on CorruptedTreeException.
(cherry picked from commit 1f746e8)
---
.../wal/scanner/PrintToFileHandler.java | 5 +-
.../persistence/wal/scanner/PrintToLogHandler.java | 6 ++-
.../persistence/wal/scanner/ScannerHandler.java | 18 +++++++
.../cache/persistence/wal/scanner/WalScanner.java | 63 +++++++++++++---------
.../diagnostic/PageHistoryDiagnoster.java | 47 ++--------------
5 files changed, 70 insertions(+), 69 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java
index 4693700..5c72bf0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java
@@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.scanner;
import java.io.File;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.nio.file.StandardOpenOption;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.DataStorageConfiguration;
@@ -29,6 +28,8 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactor
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandler.toStringRecord;
import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.DEFAULT_WAL_RECORD_PREFIX;
/**
@@ -67,7 +68,7 @@ class PrintToFileHandler implements ScannerHandler {
* @return Bytes repersentation of data to be written in dump file.
*/
protected byte[] getBytes(IgniteBiTuple<WALPointer, WALRecord> record) {
- return (DEFAULT_WAL_RECORD_PREFIX + record.get2() + "\n").getBytes(StandardCharsets.UTF_8);
+ return (DEFAULT_WAL_RECORD_PREFIX + toStringRecord(record.get2()) + System.lineSeparator()).getBytes(UTF_8);
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java
index 2d1af63..2061a44 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java
@@ -22,6 +22,7 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.lang.IgniteBiTuple;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandler.toStringRecord;
import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.DEFAULT_WAL_RECORD_PREFIX;
/**
@@ -47,7 +48,10 @@ class PrintToLogHandler implements ScannerHandler {
@Override public void handle(IgniteBiTuple<WALPointer, WALRecord> record) {
ensureNotFinished();
- resultString.append(DEFAULT_WAL_RECORD_PREFIX).append(record.get2()).append("\n");
+ resultString
+ .append(DEFAULT_WAL_RECORD_PREFIX)
+ .append(toStringRecord(record.get2()))
+ .append(System.lineSeparator());
}
/** {@inheritDoc} */
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java
index f88815c..adbfae8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java
@@ -66,4 +66,22 @@ public interface ScannerHandler {
}
};
}
+
+ /**
+ * Make string from given wal record.
+ *
+ * @param walRecord Source WAL record.
+ * @return Representation of WAL record.
+ */
+ public static String toStringRecord(WALRecord walRecord) {
+ String walRecordStr;
+
+ try {
+ walRecordStr = walRecord != null ? walRecord.toString() : "Record is null";
+ }
+ catch (RuntimeException e) {
+ walRecordStr = "Record : " + walRecord.type() + " - Unable to convert to string representation.";
+ }
+ return walRecordStr;
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java
index fe133a0..c475b91 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java
@@ -45,21 +45,45 @@ import static org.apache.ignite.internal.processors.cache.persistence.wal.reader
* Scanning WAL by specific condition.
*/
public class WalScanner {
- /** Parameters for iterator. */
- private final IteratorParametersBuilder parametersBuilder;
- /** Wal iterator factory. */
- private final IgniteWalIteratorFactory iteratorFactory;
+ /** Low level WAL iterator. */
+ private final IgniteThrowableSupplier<WALIterator> walIteratorSupplier;
/**
+ * @param preconfiguredIter Preconfgured iterator.
* @param parametersBuilder Parameters for iterator.
* @param factory Factory of iterator.
*/
WalScanner(
+ WALIterator preconfiguredIter,
IteratorParametersBuilder parametersBuilder,
IgniteWalIteratorFactory factory
) {
- this.parametersBuilder = parametersBuilder;
- iteratorFactory = factory == null ? new IgniteWalIteratorFactory() : factory;
+ if (preconfiguredIter != null)
+ walIteratorSupplier = () -> preconfiguredIter;
+ else
+ walIteratorSupplier = () -> standaloneWalIterator(
+ factory == null ? new IgniteWalIteratorFactory() : factory,
+ parametersBuilder
+ );
+ }
+
+ /**
+ * @param iteratorFactory Factory of iterator.
+ * @param parametersBuilder Parameters for iterator.
+ * @return Standalone WAL iterator created by given parameters.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static WALIterator standaloneWalIterator(
+ IgniteWalIteratorFactory iteratorFactory,
+ IteratorParametersBuilder parametersBuilder
+ ) throws IgniteCheckedException {
+ return iteratorFactory.iterator(
+ parametersBuilder.copy().addFilter((type, pointer) ->
+ // PHYSICAL need fo page shanpshot or delta record.
+ // MIXED need for partiton meta state update.
+ type.purpose() == PHYSICAL || type.purpose() == MIXED
+ )
+ );
}
/**
@@ -85,26 +109,17 @@ public class WalScanner {
.or(pageOwner(groupAndPageIds0))
.or(partitionMetaStateUpdate(groupAndParts));
- return new ScanTerminateStep(() -> iterator(filter,
- parametersBuilder.copy().addFilter((type, pointer) ->
- // PHYSICAL need fo page shanpshot or delta record.
- // MIXED need for partiton meta state update.
- type.purpose() == PHYSICAL || type.purpose() == MIXED
- )
- ));
+ return new ScanTerminateStep(() -> new FilteredWalIterator(walIteratorSupplier.get(), filter));
}
/**
- * @param filter Record filter.
- * @param parametersBuilder Iterator parameters for customization.
- * @return Instance of {@link FilteredWalIterator}.
- * @throws IgniteCheckedException If initialization of iterator will be failed.
+ * Factory method of {@link WalScanner}.
+ *
+ * @param walIterator Preconfigured WAL iterator.
+ * @return Instance of {@link WalScanner}.
*/
- @NotNull private FilteredWalIterator iterator(
- Predicate<IgniteBiTuple<WALPointer, WALRecord>> filter,
- IteratorParametersBuilder parametersBuilder
- ) throws IgniteCheckedException {
- return new FilteredWalIterator(iteratorFactory.iterator(parametersBuilder), filter);
+ public static WalScanner buildWalScanner(WALIterator walIterator) {
+ return new WalScanner(walIterator, null, null);
}
/**
@@ -114,7 +129,7 @@ public class WalScanner {
* @return Instance of {@link WalScanner}.
*/
public static WalScanner buildWalScanner(IteratorParametersBuilder parametersBuilder) {
- return new WalScanner(parametersBuilder, null);
+ return buildWalScanner(parametersBuilder, null);
}
/**
@@ -128,7 +143,7 @@ public class WalScanner {
IteratorParametersBuilder parametersBuilder,
IgniteWalIteratorFactory factory
) {
- return new WalScanner(parametersBuilder, factory);
+ return new WalScanner(null, parametersBuilder, factory);
}
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java
index 3572fc8..9d1b2fc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java
@@ -21,21 +21,14 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
@@ -50,17 +43,13 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.Re
import org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DiagnosticFileWriteMode;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.NotNull;
import static java.util.Objects.requireNonNull;
import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder.withIteratorParameters;
-import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.checkpoint;
-import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.pageOwner;
-import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.partitionMetaStateUpdate;
+import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printRawToFile;
import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToFile;
import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToLog;
-import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printRawToFile;
import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.WalScanner.buildWalScanner;
/**
@@ -199,43 +188,17 @@ public class PageHistoryDiagnoster {
ScannerHandler action,
FileWALPointer from
) throws IgniteCheckedException {
- IgniteBiTuple<WALPointer, WALRecord> lastReadRec = null;
// Try scan via WAL manager. More safety way on working node.
try {
- Set<T2<Integer, Long>> groupAndPageIds0 = new HashSet<>(builder.pageIds);
-
- // Collect all (group, partition) partition pairs.
- Set<T2<Integer, Integer>> groupAndParts = groupAndPageIds0.stream()
- .map((tup) -> new T2<>(tup.get1(), PageIdUtils.partId(tup.get2())))
- .collect(Collectors.toSet());
-
- // Build WAL filter. (Checkoint, Page, Partition meta)
- Predicate<IgniteBiTuple<WALPointer, WALRecord>> filter = checkpoint()
- .or(pageOwner(groupAndPageIds0))
- .or(partitionMetaStateUpdate(groupAndParts));
-
- try (WALIterator it = wal.replay(from)) {
- while (it.hasNext()) {
- IgniteBiTuple<WALPointer, WALRecord> recTup = lastReadRec = it.next();
-
- if (filter.test(recTup))
- action.handle(recTup);
- }
- }
- finally {
- action.finish();
- }
+ buildWalScanner(wal.replay(from))
+ .findAllRecordsFor(builder.pageIds)
+ .forEach(action);
return;
}
catch (IgniteCheckedException e) {
- if (lastReadRec != null) {
- log.warning("Failed to diagnosric scan via WAL manager, lastReadRec:["
- + lastReadRec.get1() + ", " + lastReadRec.get2() + "]",e);
- }
- else
- log.warning("Failed to diagnosric scan via WAL manager", e);
+ log.warning("Failed to diagnosric scan via WAL manager", e);
}
// Try scan via stand alone iterator is not safety if wal still generated and moving to archive.