You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ni...@apache.org on 2022/06/21 20:04:13 UTC
[ignite] branch master updated: IGNITE-17161 Exclude repeated stack trace for index-reader (#10092)
This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 3c801e37cb2 IGNITE-17161 Exclude repeated stack trace for index-reader (#10092)
3c801e37cb2 is described below
commit 3c801e37cb263da76d3c5b845bf42766ca1d62bb
Author: Nikolay <ni...@apache.org>
AuthorDate: Tue Jun 21 23:03:40 2022 +0300
IGNITE-17161 Exclude repeated stack trace for index-reader (#10092)
---
bin/index-reader.sh | 0
.../query/calcite/exec/ExecutionContext.java | 8 +-
.../common/AbstractEventSecurityContextTest.java | 2 +-
...CacheCreateDestroyEventSecurityContextTest.java | 2 +-
.../common/CacheEventSecurityContextTest.java | 2 +-
.../jdbc2/JdbcDynamicIndexAbstractSelfTest.java | 2 +-
.../ignite/jdbc/thin/JdbcThinAbstractSelfTest.java | 2 +-
.../JdbcThinConnectionMvccEnabledSelfTest.java | 2 +-
.../jdbc/thin/JdbcThinConnectionSelfTest.java | 2 +-
.../thin/JdbcThinPreparedStatementSelfTest.java | 2 +-
.../jdbc/thin/JdbcThinResultSetSelfTest.java | 2 +-
.../jdbc/thin/JdbcThinStatementSelfTest.java | 2 +-
.../internal/commandline/CommandHandler.java | 8 +-
.../argument/parser/CLIArgumentParser.java | 28 +-
.../commandline/indexreader/CacheAwareLink.java | 10 +-
.../commandline/indexreader/IgniteIndexReader.java | 1848 ++++++++------------
.../IgniteIndexReaderFilePageStoreFactory.java | 84 +-
.../IgniteIndexReaderFilePageStoreFactoryImpl.java | 103 --
.../commandline/indexreader/IndexReaderUtils.java | 95 -
.../commandline/indexreader/ItemCallback.java | 26 -
.../commandline/indexreader/PageCallback.java | 26 -
.../commandline/indexreader/PageContent.java | 47 -
.../commandline/indexreader/PageListsInfo.java | 21 +-
.../{TreeTraverseContext.java => ScanContext.java} | 56 +-
.../internal/commandline/indexreader/TreeNode.java | 47 -
.../commandline/indexreader/TreeTraversalInfo.java | 60 -
.../commandline/systemview/SystemViewCommand.java | 10 +-
.../indexreader/IgniteIndexReaderTest.java | 117 +-
.../util/GridCommandHandlerAbstractTest.java | 2 +-
.../org/apache/ignite/internal/IgnitionEx.java | 6 +-
.../internal/processors/cache/CacheObject.java | 3 -
.../apache/ignite/internal/util/IgniteUtils.java | 3 +
.../ignite/internal/util/lang/RunnableX.java | 43 +
.../ignite/internal/metric/JmxExporterSpiTest.java | 5 +-
.../CacheCreateDestroyClusterReadOnlyModeTest.java | 4 +-
.../cache/GridCacheProcessorActiveTxTest.java | 2 +-
.../distributed/CacheBlockOnReadAbstractTest.java | 2 +-
.../compute/ComputePermissionCheckTest.java | 2 +-
...cutorServiceRemoteSecurityContextCheckTest.java | 4 +-
.../AbstractContinuousQuerySandboxTest.java | 4 +-
.../security/sandbox/AbstractSandboxTest.java | 4 +-
.../security/sandbox/CacheSandboxTest.java | 2 +-
.../security/sandbox/ComputeSandboxTest.java | 6 +-
.../security/sandbox/DataStreamerSandboxTest.java | 4 +-
.../security/sandbox/EventsSandboxTest.java | 4 +-
.../security/sandbox/MessagingSandboxTest.java | 4 +-
.../security/sandbox/SchedulerSandboxTest.java | 4 +-
.../security/service/ServiceAuthorizationTest.java | 2 +-
.../apache/ignite/testframework/GridTestUtils.java | 48 +-
.../junits/logger/GridTestLog4jLoggerSelfTest.java | 2 +-
.../ignite/thread/ThreadPoolMetricsTest.java | 5 +-
.../DynamicColumnsAbstractConcurrentSelfTest.java | 2 +-
.../index/DynamicIndexAbstractBasicSelfTest.java | 2 +-
.../DynamicIndexAbstractConcurrentSelfTest.java | 2 +-
...SqlFieldTypeValidationOnKeyValueInsertTest.java | 4 +-
55 files changed, 1020 insertions(+), 1769 deletions(-)
diff --git a/bin/index-reader.sh b/bin/index-reader.sh
old mode 100644
new mode 100755
diff --git a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
index 87fe624ada3..1f09e873fb4 100644
--- a/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
+++ b/modules/calcite/src/main/java/org/apache/ignite/internal/processors/query/calcite/exec/ExecutionContext.java
@@ -41,6 +41,7 @@ import org.apache.ignite.internal.processors.query.calcite.prepare.BaseQueryCont
import org.apache.ignite.internal.processors.query.calcite.type.IgniteTypeFactory;
import org.apache.ignite.internal.processors.query.calcite.util.Commons;
import org.apache.ignite.internal.processors.query.calcite.util.TypeUtils;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.jetbrains.annotations.NotNull;
import static org.apache.ignite.internal.processors.query.calcite.util.Commons.checkRange;
@@ -290,13 +291,6 @@ public class ExecutionContext<Row> extends AbstractQueryContext implements DataC
});
}
- /** */
- @FunctionalInterface
- public interface RunnableX {
- /** */
- void run() throws Exception;
- }
-
/**
* Sets cancel flag, returns {@code true} if flag was changed by this call.
*
diff --git a/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java b/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java
index 8a6955d7bdc..a587f4d3c06 100644
--- a/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/common/AbstractEventSecurityContextTest.java
@@ -45,11 +45,11 @@ import org.apache.ignite.internal.processors.rest.GridRestCommand;
import org.apache.ignite.internal.processors.rest.protocols.http.jetty.GridJettyObjectMapper;
import org.apache.ignite.internal.processors.security.AbstractSecurityTest;
import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.plugin.security.SecurityPermissionSet;
import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Collections.singletonMap;
diff --git a/modules/clients/src/test/java/org/apache/ignite/common/CacheCreateDestroyEventSecurityContextTest.java b/modules/clients/src/test/java/org/apache/ignite/common/CacheCreateDestroyEventSecurityContextTest.java
index 438d080d2e5..e6c073f2629 100644
--- a/modules/clients/src/test/java/org/apache/ignite/common/CacheCreateDestroyEventSecurityContextTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/common/CacheCreateDestroyEventSecurityContextTest.java
@@ -33,9 +33,9 @@ import org.apache.ignite.internal.client.GridClientConfiguration;
import org.apache.ignite.internal.client.GridClientFactory;
import org.apache.ignite.internal.processors.rest.GridRestCommand;
import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.plugin.security.SecurityCredentials;
import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Test;
import static java.util.Collections.singletonList;
diff --git a/modules/clients/src/test/java/org/apache/ignite/common/CacheEventSecurityContextTest.java b/modules/clients/src/test/java/org/apache/ignite/common/CacheEventSecurityContextTest.java
index 24407688f62..59ff87ab17f 100644
--- a/modules/clients/src/test/java/org/apache/ignite/common/CacheEventSecurityContextTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/common/CacheEventSecurityContextTest.java
@@ -51,9 +51,9 @@ import org.apache.ignite.internal.client.GridClientData;
import org.apache.ignite.internal.client.GridClientDataConfiguration;
import org.apache.ignite.internal.client.GridClientFactory;
import org.apache.ignite.internal.processors.rest.GridRestCommand;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.plugin.security.SecurityCredentials;
import org.apache.ignite.plugin.security.SecurityCredentialsBasicProvider;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java
index 8d0c0c4afda..970da2e00d8 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/jdbc2/JdbcDynamicIndexAbstractSelfTest.java
@@ -30,8 +30,8 @@ import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Test;
/**
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java
index 8d089f427a8..a44596f146d 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinAbstractSelfTest.java
@@ -32,9 +32,9 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.index.AbstractIndexingCommonTest;
import org.apache.ignite.internal.processors.odbc.ClientListenerProcessor;
import org.apache.ignite.internal.processors.port.GridPortRecord;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
/**
* Connection test.
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionMvccEnabledSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionMvccEnabledSelfTest.java
index d99c06fd0e7..76048c424ba 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionMvccEnabledSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionMvccEnabledSelfTest.java
@@ -27,6 +27,7 @@ import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.binary.BinaryMarshaller;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.testframework.GridStringLogger;
import org.jetbrains.annotations.NotNull;
import org.junit.Test;
@@ -36,7 +37,6 @@ import static java.sql.Connection.TRANSACTION_READ_COMMITTED;
import static java.sql.Connection.TRANSACTION_READ_UNCOMMITTED;
import static java.sql.Connection.TRANSACTION_REPEATABLE_READ;
import static java.sql.Connection.TRANSACTION_SERIALIZABLE;
-import static org.apache.ignite.testframework.GridTestUtils.RunnableX;
import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
/**
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
index e0dd41462d4..91e7c87559c 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinConnectionSelfTest.java
@@ -51,6 +51,7 @@ import org.apache.ignite.internal.jdbc.thin.ConnectionPropertiesImpl;
import org.apache.ignite.internal.jdbc.thin.JdbcThinConnection;
import org.apache.ignite.internal.jdbc.thin.JdbcThinTcpIo;
import org.apache.ignite.internal.util.HostAndPortRange;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.logger.NullLogger;
@@ -73,7 +74,6 @@ import static java.sql.Statement.NO_GENERATED_KEYS;
import static java.sql.Statement.RETURN_GENERATED_KEYS;
import static org.apache.ignite.configuration.ClientConnectorConfiguration.DFLT_PORT;
import static org.apache.ignite.internal.processors.odbc.SqlStateCode.TRANSACTION_STATE_EXCEPTION;
-import static org.apache.ignite.testframework.GridTestUtils.RunnableX;
import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
import static org.apache.ignite.testframework.GridTestUtils.getFieldValue;
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java
index 8dce9e05b4b..2becc186082 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinPreparedStatementSelfTest.java
@@ -44,9 +44,9 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcThinFeature;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Assert;
import org.junit.Test;
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinResultSetSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinResultSetSelfTest.java
index 4f7e759b981..50f1c1079d1 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinResultSetSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinResultSetSelfTest.java
@@ -41,13 +41,13 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.junit.Assert;
import org.junit.Test;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.testframework.GridTestUtils.RunnableX;
import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause;
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStatementSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStatementSelfTest.java
index 37f6cb1ef84..ff9a60922e8 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStatementSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinStatementSelfTest.java
@@ -33,10 +33,10 @@ import org.apache.ignite.cache.query.annotations.QuerySqlFunction;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.processors.odbc.SqlStateCode;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Ignore;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index a11e57b0c26..d24bb2960f4 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -144,13 +144,13 @@ public class CommandHandler {
/**
* @return prepared JULs logger.
*/
- private Logger setupJavaLogger() {
- Logger result = initLogger(CommandHandler.class.getName() + "Log");
+ public static Logger setupJavaLogger(String appName, Class<?> cls) {
+ Logger result = initLogger(cls.getName() + "Log");
// Adding logging to file.
try {
String absPathPattern =
- new File(JavaLoggerFileHandler.logDirectory(U.defaultWorkDirectory()), "control-utility-%g.log").getAbsolutePath();
+ new File(JavaLoggerFileHandler.logDirectory(U.defaultWorkDirectory()), appName + "-%g.log").getAbsolutePath();
FileHandler fileHandler = new FileHandler(absPathPattern, 5 * 1024 * 1024, 5);
@@ -202,7 +202,7 @@ public class CommandHandler {
*
*/
public CommandHandler() {
- logger = setupJavaLogger();
+ logger = setupJavaLogger("control-utility", CommandHandler.class);
}
/**
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java
index 6c616198650..9bf30eff2b9 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/argument/parser/CLIArgumentParser.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.commandline.argument.parser;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -84,19 +85,18 @@ public class CLIArgumentParser {
/** */
private <T> T parseVal(String val, Class<T> type) {
- switch (type.getSimpleName()) {
- case "String": return (T)val;
-
- case "String[]": return (T)val.split(",");
-
- case "Integer": return (T)wrapNumberFormatException(() -> Integer.parseInt(val), val, Integer.class);
-
- case "Long": return (T)wrapNumberFormatException(() -> Long.parseLong(val), val, Long.class);
-
- case "UUID": return (T)UUID.fromString(val);
-
- default: throw new IgniteException("Unsupported argument type: " + type.getName());
- }
+ if (type == String.class)
+ return (T)val;
+ else if (type == String[].class)
+ return (T)val.split(",");
+ else if (type == Integer.class)
+ return (T)wrapNumberFormatException(() -> Integer.parseInt(val), val, Integer.class);
+ else if (type == Long.class)
+ return (T)wrapNumberFormatException(() -> Long.parseLong(val), val, Long.class);
+ else if (type == UUID.class)
+ return (T)UUID.fromString(val);
+
+ throw new IgniteException("Unsupported argument type: " + type.getName());
}
/**
@@ -172,7 +172,7 @@ public class CLIArgumentParser {
sb.a("\n\n").a(arg.name()).a(": ").a(arg.usage());
if (arg.optional())
- sb.a(" Default value: ").a(dfltVal);
+ sb.a(" Default value: ").a(dfltVal instanceof String[] ? Arrays.toString((Object[])dfltVal) : dfltVal);
}
return sb.toString();
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/CacheAwareLink.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/CacheAwareLink.java
index 1916bba03dc..694819fe02b 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/CacheAwareLink.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/CacheAwareLink.java
@@ -22,18 +22,14 @@ package org.apache.ignite.internal.commandline.indexreader;
*/
class CacheAwareLink {
/** */
- public final int cacheId;
+ final int cacheId;
/** */
- public final long link;
-
- /** True if a link points to tombstone value. */
- public final boolean tombstone;
+ final long link;
/** */
- public CacheAwareLink(int cacheId, long link, boolean tombstone) {
+ public CacheAwareLink(int cacheId, long link) {
this.cacheId = cacheId;
this.link = link;
- this.tombstone = tombstone;
}
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java
index 4ac13c6357d..bfd51d1872a 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReader.java
@@ -18,10 +18,7 @@
package org.apache.ignite.internal.commandline.indexreader;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.util.ArrayList;
@@ -35,10 +32,9 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;
import java.util.function.Supplier;
+import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.IntStream;
@@ -47,17 +43,16 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.cache.query.index.IndexProcessor;
import org.apache.ignite.internal.cache.query.index.sorted.inline.io.InlineIO;
+import org.apache.ignite.internal.commandline.CommandHandler;
import org.apache.ignite.internal.commandline.ProgressPrinter;
-import org.apache.ignite.internal.commandline.StringBuilderOutputStream;
-import org.apache.ignite.internal.commandline.argument.parser.CLIArgument;
import org.apache.ignite.internal.commandline.argument.parser.CLIArgumentParser;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.commandline.systemview.SystemViewCommand;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
import org.apache.ignite.internal.processors.cache.persistence.StorageException;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreV2;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.AbstractDataPageIO;
@@ -73,12 +68,13 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDat
import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO;
import org.apache.ignite.internal.processors.cache.tree.PendingRowIO;
import org.apache.ignite.internal.processors.cache.tree.RowLinkIO;
-import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccDataLeafIO;
import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.GridStringBuilder;
-import org.apache.ignite.internal.util.lang.GridClosure3;
+import org.apache.ignite.internal.util.lang.GridPlainClosure2;
import org.apache.ignite.internal.util.lang.IgnitePair;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.jetbrains.annotations.Nullable;
@@ -90,15 +86,9 @@ import static java.util.Objects.isNull;
import static java.util.Objects.nonNull;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toList;
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_PAGE_SIZE;
import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.mandatoryArg;
import static org.apache.ignite.internal.commandline.argument.parser.CLIArgument.optionalArg;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.CHECK_PARTS;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.DEST_FILE;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.DIR;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.INDEXES;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.PAGE_SIZE;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.PAGE_STORE_VER;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.Args.PART_CNT;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
@@ -108,18 +98,18 @@ import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
import static org.apache.ignite.internal.pagemem.PageIdUtils.pageIndex;
import static org.apache.ignite.internal.pagemem.PageIdUtils.partId;
import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
-import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getType;
-import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getVersion;
import static org.apache.ignite.internal.util.GridUnsafe.allocateBuffer;
import static org.apache.ignite.internal.util.GridUnsafe.bufferAddress;
import static org.apache.ignite.internal.util.GridUnsafe.freeBuffer;
+import static org.apache.ignite.internal.visor.systemview.VisorSystemViewTask.SimpleType.NUMBER;
+import static org.apache.ignite.internal.visor.systemview.VisorSystemViewTask.SimpleType.STRING;
/**
* Offline reader for index files.
*/
public class IgniteIndexReader implements AutoCloseable {
/** */
- private static final String META_TREE_NAME = "MetaTree";
+ public static final String META_TREE_NAME = "MetaTree";
/** */
public static final String RECURSIVE_TRAVERSE_NAME = "<RECURSIVE> ";
@@ -133,6 +123,24 @@ public class IgniteIndexReader implements AutoCloseable {
/** */
public static final String ERROR_PREFIX = "<ERROR> ";
+ /** */
+ private static final String DIR_ARG = "--dir";
+
+ /** */
+ private static final String PART_CNT_ARG = "--part-cnt";
+
+ /** */
+ private static final String PAGE_SIZE_ARG = "--page-size";
+
+ /** */
+ private static final String PAGE_STORE_VER_ARG = "--page-store-ver";
+
+ /** */
+ private static final String INDEXES_ARG = "--indexes";
+
+ /** */
+ private static final String CHECK_PARTS_ARG = "--check-parts";
+
/** */
private static final Pattern CACHE_TYPE_ID_SEARCH_PATTERN =
Pattern.compile("(?<id>[-0-9]{1,15})_(?<typeId>[-0-9]{1,15})_.*");
@@ -144,10 +152,6 @@ public class IgniteIndexReader implements AutoCloseable {
/** */
private static final int CHECK_PARTS_MAX_ERRORS_PER_PARTITION = 10;
- /** */
- private static final Map<String, IgnitePair<Integer>> CACHE_TYPE_IDS = new HashMap<>();
-
- /** */
static {
IndexProcessor.registerIO();
}
@@ -158,443 +162,436 @@ public class IgniteIndexReader implements AutoCloseable {
/** Partition count. */
private final int partCnt;
+ /** Check cache data tree in partition files and it's consistency with indexes. */
+ private final boolean checkParts;
+
/** Index name filter, if {@code null} then is not used. */
@Nullable private final Predicate<String> idxFilter;
- /** Output strean. */
- private final PrintStream outStream;
+ /** Logger. */
+ private final Logger log;
/** Page store of {@link FilePageStoreManager#INDEX_FILE_NAME}. */
- @Nullable private final FilePageStore idxStore;
+ private final FilePageStore idxStore;
/** Partitions page stores, may contains {@code null}. */
- @Nullable private final FilePageStore[] partStores;
-
- /** Check cache data tree in partition files and it's consistency with indexes. */
- private final boolean checkParts;
+ private final FilePageStore[] partStores;
/** */
private final Set<Integer> missingPartitions = new HashSet<>();
/** */
- private final PageIOProcessor innerPageIOProcessor = new InnerPageIOProcessor();
+ private final Set<Long> pageIds = new HashSet<>();
+
+ /** */
+ private final InnerPageVisitor innerPageVisitor = new InnerPageVisitor();
+
+ /** */
+ private final LeafPageVisitor leafPageVisitor = new LeafPageVisitor();
+
+ /** */
+ private final MetaPageVisitor metaPageVisitor = new MetaPageVisitor();
/** */
- private final PageIOProcessor leafPageIOProcessor = new LeafPageIOProcessor();
+ private final LevelsPageVisitor levelsPageVisitor = new LevelsPageVisitor();
/** */
- private final PageIOProcessor metaPageIOProcessor = new MetaPageIOProcessor();
+ private final Map<String, IgnitePair<Integer>> cacheTypeIds = new HashMap<>();
/**
* Constructor.
*
* @param idxFilter Index name filter, if {@code null} then is not used.
* @param checkParts Check cache data tree in partition files and it's consistency with indexes.
- * @param outStream {@link PrintStream} for print report, if {@code null} then will be used {@link System#out}.
+ * @param filePageStoreFactory File page store factory.
+ * @param log Logger.
* @throws IgniteCheckedException If failed.
*/
public IgniteIndexReader(
@Nullable Predicate<String> idxFilter,
boolean checkParts,
- @Nullable PrintStream outStream,
- IgniteIndexReaderFilePageStoreFactory filePageStoreFactory
+ IgniteIndexReaderFilePageStoreFactory filePageStoreFactory,
+ Logger log
) throws IgniteCheckedException {
pageSize = filePageStoreFactory.pageSize();
partCnt = filePageStoreFactory.partitionCount();
this.checkParts = checkParts;
this.idxFilter = idxFilter;
-
- this.outStream = isNull(outStream) ? System.out : outStream;
-
- Map<Integer, List<Throwable>> partStoresErrors = new HashMap<>();
- List<Throwable> errors = new ArrayList<>();
-
- idxStore = filePageStoreFactory.createFilePageStoreWithEnsure(INDEX_PARTITION, FLAG_IDX, errors);
-
- if (!errors.isEmpty())
- partStoresErrors.put(INDEX_PARTITION, new ArrayList<>(errors));
+ this.log = log;
+ idxStore = filePageStoreFactory.createFilePageStore(INDEX_PARTITION, FLAG_IDX);
if (isNull(idxStore))
throw new IgniteCheckedException(INDEX_FILE_NAME + " file not found");
- else
- print("Analyzing file: " + INDEX_FILE_NAME);
- partStores = new FilePageStore[partCnt];
-
- for (int i = 0; i < partCnt; i++) {
- if (!errors.isEmpty())
- errors.clear();
-
- partStores[i] = filePageStoreFactory.createFilePageStoreWithEnsure(i, FLAG_DATA, errors);
+ log.info("Analyzing file: " + INDEX_FILE_NAME);
- if (!errors.isEmpty())
- partStoresErrors.put(i, new ArrayList<>(errors));
- }
+ partStores = new FilePageStore[partCnt];
- printFileReadingErrors(partStoresErrors);
+ for (int i = 0; i < partCnt; i++)
+ partStores[i] = filePageStoreFactory.createFilePageStore(i, FLAG_DATA);
}
- /** */
- private void print(String s) {
- outStream.println(s);
- }
+ /**
+ * Entry point.
+ *
+ * @param args Arguments.
+ */
+ public static void main(String[] args) {
+ System.out.println("THIS UTILITY MUST BE LAUNCHED ON PERSISTENT STORE WHICH IS NOT UNDER RUNNING GRID!");
- /** */
- private void printErr(String s) {
- outStream.println(ERROR_PREFIX + s);
- }
+ CLIArgumentParser p = new CLIArgumentParser(asList(
+ mandatoryArg(
+ DIR_ARG,
+ "partition directory, where " + INDEX_FILE_NAME + " and (optionally) partition files are located.",
+ String.class
+ ),
+ optionalArg(PART_CNT_ARG, "full partitions count in cache group.", Integer.class, () -> 0),
+ optionalArg(PAGE_SIZE_ARG, "page size.", Integer.class, () -> DFLT_PAGE_SIZE),
+ optionalArg(PAGE_STORE_VER_ARG, "page store version.", Integer.class, () -> FilePageStoreV2.VERSION),
+ optionalArg(INDEXES_ARG, "you can specify index tree names that will be processed, separated by comma " +
+ "without spaces, other index trees will be skipped.", String[].class, () -> U.EMPTY_STRS),
+ optionalArg(CHECK_PARTS_ARG,
+ "check cache data tree in partition files and it's consistency with indexes.", Boolean.class, () -> false)
+ ));
- /** */
- private void printErrors(
- String prefix,
- String caption,
- @Nullable String alternativeCaption,
- String elementFormatPtrn,
- boolean printTrace,
- Map<?, ? extends List<? extends Throwable>> errors
- ) {
- if (errors.isEmpty() && alternativeCaption != null) {
- print(prefix + alternativeCaption);
+ if (args.length == 0) {
+ System.out.println(p.usage());
return;
}
- if (caption != null)
- outStream.println(prefix + ERROR_PREFIX + caption);
-
- errors.forEach((k, v) -> {
- outStream.println(prefix + ERROR_PREFIX + format(elementFormatPtrn, k.toString()));
+ p.parse(asList(args).iterator());
- v.forEach(e -> {
- if (printTrace)
- printStackTrace(e);
- else
- printErr(e.getMessage());
- });
- });
- }
+ IgniteIndexReaderFilePageStoreFactory filePageStoreFactory = new IgniteIndexReaderFilePageStoreFactory(
+ new File(p.<String>get(DIR_ARG)),
+ p.get(PAGE_SIZE_ARG),
+ p.get(PART_CNT_ARG),
+ p.get(PAGE_STORE_VER_ARG)
+ );
- /** */
- private void printPageStat(String prefix, String caption, Map<Class<? extends PageIO>, Long> stat) {
- if (caption != null)
- print(prefix + caption + (stat.isEmpty() ? " empty" : ""));
+ Set<String> idxs = new HashSet<>(asList(p.get(INDEXES_ARG)));
- stat.forEach((cls, cnt) -> print(prefix + cls.getSimpleName() + ": " + cnt));
+ try (IgniteIndexReader reader = new IgniteIndexReader(
+ idxs.isEmpty() ? null : idxs::contains,
+ p.get(CHECK_PARTS_ARG),
+ filePageStoreFactory,
+ CommandHandler.setupJavaLogger("index-reader", IgniteIndexReader.class)
+ )) {
+ reader.readIndex();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(INDEX_FILE_NAME + " scan problem", e);
+ }
}
- /** */
- private void printStackTrace(Throwable e) {
- OutputStream os = new StringBuilderOutputStream();
+ /** Read index file. */
+ public void readIndex() throws IgniteCheckedException {
+ log.info("Partitions files num: " + Arrays.stream(partStores).filter(Objects::nonNull).count());
+ log.info("Going to check " + ((idxStore.size() - idxStore.headerSize()) / pageSize) + " pages.");
- e.printStackTrace(new PrintStream(os));
+ long[] indexPartitionRoots = partitionRoots(PageIdAllocator.META_PAGE_ID);
- outStream.println(os);
- }
+ Map<String, ScanContext> recursiveScans = scanAllTrees(
+ "Index trees traversal",
+ indexPartitionRoots[0],
+ CountOnlyStorage::new,
+ this::recursiveTreeScan
+ );
- /** */
- static long normalizePageId(long pageId) {
- return pageId(partId(pageId), flag(pageId), pageIndex(pageId));
- }
+ Map<String, ScanContext> horizontalScans = scanAllTrees(
+ "Scan index trees horizontally",
+ indexPartitionRoots[0],
+ checkParts ? LinkStorage::new : CountOnlyStorage::new,
+ this::horizontalTreeScan
+ );
- /**
- * Reading pages into buffer.
- *
- * @param store Source for reading pages.
- * @param pageId Page ID.
- * @param buf Buffer.
- */
- private void readPage(FilePageStore store, long pageId, ByteBuffer buf) throws IgniteCheckedException {
- try {
- store.read(pageId, buf, false);
- }
- catch (IgniteDataIntegrityViolationException | IllegalArgumentException e) {
- // Replacing exception due to security reasons, as IgniteDataIntegrityViolationException prints page content.
- // Catch IllegalArgumentException for output page information.
- throw new IgniteException("Failed to read page, id=" + pageId + ", idx=" + pageIndex(pageId) +
- ", file=" + store.getFileAbsolutePath());
- }
- }
+ printScanResults(RECURSIVE_TRAVERSE_NAME, recursiveScans);
+ printScanResults(HORIZONTAL_SCAN_NAME, horizontalScans);
- /**
- * @return Tuple consisting of meta tree root page and pages list root page.
- * @throws IgniteCheckedException If failed.
- */
- IgniteBiTuple<Long, Long> partitionRoots(long pageMetaPageId) throws IgniteCheckedException {
- return doWithBuffer((buf, addr) -> {
- readPage(filePageStore(partId(pageMetaPageId)), pageMetaPageId, buf);
+ compareScans(recursiveScans, horizontalScans);
- PageMetaIO pageMetaIO = PageIO.getPageIO(addr);
+ printPagesListsInfo(indexPartitionRoots[1]);
- return new IgniteBiTuple<>(
- normalizePageId(pageMetaIO.getTreeRoot(addr)),
- normalizePageId(pageMetaIO.getReuseListRoot(addr))
- );
- });
- }
+ printSequentialScanInfo(scanIndexSequentially());
- /**
- * Read index file.
- */
- public void readIdx() throws IgniteCheckedException {
- long partPageStoresNum = Arrays.stream(partStores)
- .filter(Objects::nonNull)
- .count();
+ if (checkParts)
+ printCheckPartsInfo(checkParts(horizontalScans));
+ }
- print("Partitions files num: " + partPageStoresNum);
+ /** Traverse all trees in file and return their info. */
+ private Map<String, ScanContext> scanAllTrees(
+ String caption,
+ long metaTreeRoot,
+ Supplier<ItemStorage> itemStorageFactory,
+ Scanner scanner
+ ) {
+ Map<String, ScanContext> ctxs = new LinkedHashMap<>();
- Map<Class<? extends PageIO>, Long> pageClasses = new HashMap<>();
+ ScanContext metaTreeCtx = scanner.scan(metaTreeRoot, META_TREE_NAME, new ItemsListStorage<IndexStorageImpl.IndexItem>());
- long pagesNum = isNull(idxStore) ? 0 : (idxStore.size() - idxStore.headerSize()) / pageSize;
+ ctxs.put(META_TREE_NAME, metaTreeCtx);
- print("Going to check " + pagesNum + " pages.");
+ ProgressPrinter progressPrinter = createProgressPrinter(caption, metaTreeCtx.items.size());
- Set<Long> pageIds = new HashSet<>();
+ ((ItemsListStorage<IndexStorageImpl.IndexItem>)metaTreeCtx.items).forEach(item -> {
+ progressPrinter.printProgress();
- IgniteBiTuple<Long, Long> indexPartitionRoots = partitionRoots(partMetaPageId(INDEX_PARTITION, FLAG_IDX));
+ if (nonNull(idxFilter) && !idxFilter.test(item.nameString()))
+ return;
- long metaTreeRootId = indexPartitionRoots.get1();
- long pageListMetaPageId = indexPartitionRoots.get2();
+ ScanContext ctx =
+ scanner.scan(normalizePageId(item.pageId()), item.nameString(), itemStorageFactory.get());
- // Traversing trees.
- Map<String, TreeTraversalInfo> treeInfo =
- traverseAllTrees("Index trees traversal", metaTreeRootId, CountOnlyStorage::new, this::traverseTree);
+ ctxs.put(item.toString(), ctx);
+ });
- treeInfo.forEach((name, info) -> {
- pageIds.addAll(info.innerPageIds);
+ return ctxs;
+ }
- pageIds.add(info.rootPageId);
- });
+ /**
+ * Traverse single index tree from root to leafs.
+ *
+ * @param rootPageId Root page id.
+ * @param idx Index name.
+ * @param items Items storage.
+ * @return Tree traversal context.
+ */
+ ScanContext recursiveTreeScan(long rootPageId, String idx, ItemStorage items) {
+ pageIds.add(normalizePageId(rootPageId));
- Supplier<ItemStorage> itemStorageFactory = checkParts ? LinkStorage::new : CountOnlyStorage::new;
+ ScanContext ctx = createContext(cacheAndTypeId(idx).get1(), filePageStore(rootPageId), items);
- Map<String, TreeTraversalInfo> horizontalScans =
- traverseAllTrees("Scan index trees horizontally", metaTreeRootId, itemStorageFactory, this::horizontalTreeScan);
+ metaPageVisitor.readAndVisit(rootPageId, ctx);
- // Scanning page reuse lists.
- PageListsInfo pageListsInfo = pageListMetaPageId == 0 ? null : getPageListsInfo(pageListMetaPageId);
+ return ctx;
+ }
- ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Reading pages sequentially", pagesNum);
+ /**
+ * Traverse single index tree by each level horizontally.
+ *
+ * @param rootPageId Root page id.
+ * @param idx Index name.
+ * @param items Items storage.
+ * @return Tree traversal context.
+ */
+ private ScanContext horizontalTreeScan(long rootPageId, String idx, ItemStorage items) {
+ pageIds.add(normalizePageId(rootPageId));
- // Scan all pages in file.
- List<Throwable> errors = scanFileStore(INDEX_PARTITION, FLAG_IDX, idxStore, (pageId, addr, io) -> {
- progressPrinter.printProgress();
+ ScanContext ctx = createContext(cacheAndTypeId(idx).get1(), filePageStore(rootPageId), items);
- pageClasses.compute(io.getClass(), (k, v) -> v == null ? 1 : v + 1);
-
- if (!(io instanceof PageMetaIO || io instanceof PagesListMetaIO)) {
- if (idxFilter == null) {
- if ((io instanceof BPlusMetaIO || io instanceof BPlusInnerIO)
- && !pageIds.contains(pageId)
- && pageListsInfo != null
- && !pageListsInfo.allPages.contains(pageId)) {
- throw new IgniteException(
- "Possibly orphan " + io.getClass().getSimpleName() + " page, pageId=" + pageId
- );
- }
- }
- }
+ levelsPageVisitor.readAndVisit(rootPageId, ctx);
- return true;
- });
+ return ctx;
+ }
- printTraversalResults(RECURSIVE_TRAVERSE_NAME, treeInfo);
+ /**
+ * Gets info about page lists.
+ *
+ * @param metaPageListId Page list meta id.
+ * @return Page list info.
+ */
+ private PageListsInfo pageListsInfo(long metaPageListId) throws IgniteCheckedException {
+ return doWithBuffer((buf, addr) -> {
+ Map<IgniteBiTuple<Long, Integer>, List<Long>> bucketsData = new HashMap<>();
- printTraversalResults(HORIZONTAL_SCAN_NAME, horizontalScans);
+ Map<Class<? extends PageIO>, Long> ioStat = new HashMap<>();
- compareTraversals(treeInfo, horizontalScans);
+ Map<Long, List<String>> errors = new HashMap<>();
- if (pageListsInfo == null)
- printErr("No page lists meta info found.");
- else
- printPagesListsInfo(pageListsInfo);
+ long pagesCnt = 0;
+ long currMetaPageId = metaPageListId;
- printPageStat("", "\n---These pages types were encountered during sequential scan:", pageClasses);
+ while (currMetaPageId != 0) {
+ try {
+ PagesListMetaIO io = readPage(idxStore, currMetaPageId, buf);
- if (!errors.isEmpty()) {
- printErr("---");
- printErr("Errors:");
+ pageIds.add(normalizePageId(currMetaPageId));
- errors.forEach(this::printStackTrace);
- }
+ Map<Integer, GridLongList> data = new HashMap<>();
- print("---");
- print("Total pages encountered during sequential scan: " + pageClasses.values().stream().mapToLong(a -> a).sum());
- print("Total errors occurred during sequential scan: " + errors.size());
+ io.getBucketsData(addr, data);
- if (idxFilter != null)
- print("Orphan pages were not reported due to --indexes filter.");
+ for (Map.Entry<Integer, GridLongList> e : data.entrySet()) {
+ List<Long> listIds = LongStream.of(e.getValue().array())
+ .map(IgniteIndexReader::normalizePageId)
+ .boxed()
+ .collect(toList());
- print("Note that some pages can be occupied by meta info, tracking info, etc., so total page count can differ " +
- "from count of pages found in index trees and page lists.");
+ for (Long listId : listIds) {
+ try {
+ pagesCnt += visitPageList(listId, ioStat);
+ }
+ catch (Exception err) {
+ errors.put(listId, singletonList(err.getMessage()));
+ }
+ }
- if (checkParts) {
- Map<Integer, List<Throwable>> checkPartsErrors = checkParts(horizontalScans);
+ bucketsData.put(F.t(currMetaPageId, e.getKey()), listIds);
+ }
- print("");
+ currMetaPageId = io.getNextMetaPageId(addr);
+ }
+ catch (Exception e) {
+ errors.put(currMetaPageId, singletonList(e.getMessage()));
- printErrors("",
- "Partitions check:",
- "Partitions check detected no errors.",
- "Errors detected in partition, partId=%s",
- false,
- checkPartsErrors
- );
+ break;
+ }
+ }
- print("\nPartition check finished, total errors: " +
- checkPartsErrors.values().stream().mapToInt(List::size).sum() + ", total problem partitions: " +
- checkPartsErrors.size()
- );
- }
+ return new PageListsInfo(bucketsData, pagesCnt, ioStat, errors);
+ });
}
/**
- * Print partitions reading exceptions.
+ * Visit single page list.
*
- * @param partStoresErrors Partitions reading exceptions.
+ * @param listStartPageId Id of the start page of the page list.
+ * @param ioStat Page types statistics.
+ * @return List of page ids.
*/
- private void printFileReadingErrors(Map<Integer, List<Throwable>> partStoresErrors) {
- List<Throwable> idxPartErrors = partStoresErrors.get(INDEX_PARTITION);
+ private long visitPageList(long listStartPageId, Map<Class<? extends PageIO>, Long> ioStat) throws IgniteCheckedException {
+ return doWithBuffer((nodeBuf, nodeAddr) -> doWithBuffer((pageBuf, pageAddr) -> {
+ long res = 0;
- if (!F.isEmpty(idxPartErrors)) {
- printErr("Errors detected while reading " + INDEX_FILE_NAME);
+ long currPageId = listStartPageId;
- idxPartErrors.forEach(err -> printErr(err.getMessage()));
+ while (currPageId != 0) {
+ PagesListNodeIO io = readPage(idxStore, currPageId, nodeBuf);
- partStoresErrors.remove(INDEX_PARTITION);
- }
+ ScanContext.onPageIO(readPage(idxStore, currPageId, pageBuf).getClass(), ioStat, 1);
- if (!partStoresErrors.isEmpty()) {
- printErrors("", "Errors detected while reading partition files:", null,
- "Partition id: %s, exceptions: ", false, partStoresErrors);
- }
- }
+ pageIds.add(normalizePageId(normalizePageId(currPageId)));
- /**
- * Allocates buffer and does some work in closure, then frees the buffer.
- *
- * @param c Closure.
- * @param <T> Result type.
- * @return Result of closure.
- * @throws IgniteCheckedException If failed.
- */
- private <T> T doWithBuffer(BufferClosure<T> c) throws IgniteCheckedException {
- ByteBuffer buf = allocateBuffer(pageSize);
+ res += io.getCount(nodeAddr);
- try {
- long addr = bufferAddress(buf);
+ for (int i = 0; i < io.getCount(nodeAddr); i++) {
+ long pageId = normalizePageId(io.getAt(nodeAddr, i));
- return c.apply(buf, addr);
- }
- finally {
- freeBuffer(buf);
- }
+ pageIds.add(pageId);
+
+ ScanContext.onPageIO(readPage(idxStore, pageId, pageBuf).getClass(), ioStat, 1);
+ }
+
+ currPageId = io.getNextId(nodeAddr);
+ }
+
+ return res;
+ }));
}
/**
- * Scans given file page store and executes closure for each page.
+ * Traverse index file sequentially.
*
- * @param partId Partition id.
- * @param flag Flag.
- * @param store Page store.
- * @param c Closure that accepts page id, page address, page IO. If it returns false, scan stops.
- * @return List of errors that occured while scanning.
+ * @return Traverse results.
* @throws IgniteCheckedException If failed.
*/
- private List<Throwable> scanFileStore(int partId, byte flag, FilePageStore store, GridClosure3<Long, Long, PageIO, Boolean> c)
- throws IgniteCheckedException {
- return doWithBuffer((buf, addr) -> {
- List<Throwable> errors = new ArrayList<>();
+ private ScanContext scanIndexSequentially() throws IgniteCheckedException {
+ long pagesNum = (idxStore.size() - idxStore.headerSize()) / pageSize;
+
+ ProgressPrinter progressPrinter = createProgressPrinter("Reading pages sequentially", pagesNum);
- long pagesNum = isNull(store) ? 0 : (store.size() - store.headerSize()) / pageSize;
+ ScanContext ctx = createContext(-1, idxStore, new CountOnlyStorage());
+ doWithBuffer((buf, addr) -> {
for (int i = 0; i < pagesNum; i++) {
- buf.rewind();
+ long pageId = -1;
try {
- long pageId = PageIdUtils.pageId(partId, flag, i);
+ pageId = pageId(INDEX_PARTITION, FLAG_IDX, i);
+
+ PageIO io = readPage(ctx, pageId, buf);
+
+ progressPrinter.printProgress();
- readPage(store, pageId, buf);
+ if (idxFilter != null)
+ continue;
- PageIO io = PageIO.getPageIO(addr);
+ if (io instanceof PageMetaIO || io instanceof PagesListMetaIO)
+ continue;
- if (!c.apply(pageId, addr, io))
- break;
+ if (!((io instanceof BPlusMetaIO || io instanceof BPlusInnerIO)))
+ continue;
+
+ if (pageIds.contains(normalizePageId(pageId)))
+ continue;
+
+ ctx.errors.put(pageId, Collections.singletonList("Error [step=" + i +
+ ", msg=Possibly orphan " + io.getClass().getSimpleName() + " page" +
+ ", pageId=" + normalizePageId(pageId) + ']'));
}
catch (Throwable e) {
- String err = "Exception occurred on step " + i + ": " + e.getMessage();
-
- errors.add(new IgniteException(err, e));
+ ctx.errors.put(
+ pageId,
+ Collections.singletonList("Error [step=" + i + ", msg=" + e.getMessage() + ']')
+ );
}
}
- return errors;
+ return null;
});
+
+ return ctx;
}
/**
- * Checks partitions, comparing partition indexes (cache data tree) to indexes given in {@code aTreesInfo}.
+ * Checks partitions, comparing partition indexes (cache data tree) to indexes given in {@code treesInfo}.
*
- * @param aTreesInfo Index trees info to compare cache data tree with.
+ * @param treesInfo Index trees info to compare cache data tree with.
* @return Map of errors, bound to partition id.
*/
- private Map<Integer, List<Throwable>> checkParts(Map<String, TreeTraversalInfo> aTreesInfo) {
- System.out.println();
+ private Map<Integer, List<String>> checkParts(Map<String, ScanContext> treesInfo) {
+ log.info("");
// Map partId -> errors.
- Map<Integer, List<Throwable>> res = new HashMap<>();
-
- Map<String, TreeTraversalInfo> treesInfo = new HashMap<>(aTreesInfo);
+ Map<Integer, List<String>> res = new HashMap<>();
- treesInfo.remove(META_TREE_NAME);
+ ProgressPrinter progressPrinter = createProgressPrinter("Checking partitions", partCnt);
- ProgressPrinter progressPrinter = new ProgressPrinter(System.out, "Checking partitions", partCnt);
-
- for (int i = 0; i < partCnt; i++) {
+ IntStream.range(0, partCnt).forEach(partId -> {
progressPrinter.printProgress();
- FilePageStore partStore = partStores[i];
+ FilePageStore partStore = partStores[partId];
if (partStore == null)
- continue;
-
- List<Throwable> errors = new LinkedList<>();
+ return;
- final int partId = i;
+ List<String> errors = new LinkedList<>();
try {
- long partMetaId = partMetaPageId(i, FLAG_DATA);
+ long partMetaId = pageId(partId, FLAG_DATA, 0);
doWithBuffer((buf, addr) -> {
- readPage(partStore, partMetaId, buf);
-
- PagePartitionMetaIO partMetaIO = PageIO.getPageIO(addr);
+ PagePartitionMetaIO partMetaIO = readPage(partStore, partMetaId, buf);
long cacheDataTreeRoot = partMetaIO.getTreeRoot(addr);
- TreeTraversalInfo cacheDataTreeInfo =
- horizontalTreeScan(cacheDataTreeRoot, "dataTree-" + partId, new ItemsListStorage());
+ ScanContext ctx =
+ horizontalTreeScan(cacheDataTreeRoot, "dataTree-" + partId, new ItemsListStorage<>());
- for (Object dataTreeItem : cacheDataTreeInfo.itemStorage) {
+ for (Object dataTreeItem : ctx.items) {
CacheAwareLink cacheAwareLink = (CacheAwareLink)dataTreeItem;
- for (Map.Entry<String, TreeTraversalInfo> e : treesInfo.entrySet()) {
- String name = e.getKey();
+ for (Map.Entry<String, ScanContext> e : treesInfo.entrySet()) {
+ if (e.getKey().equals(META_TREE_NAME))
+ continue;
- TreeTraversalInfo tree = e.getValue();
+ if (cacheAndTypeId(e.getKey()).get1() != cacheAwareLink.cacheId
+ || e.getValue().items.contains(cacheAwareLink))
+ continue;
- int cacheId = getCacheId(name);
+ long pageId = pageId(cacheAwareLink.link);
- if (cacheId != cacheAwareLink.cacheId)
- continue; // It's index for other cache, don't check.
-
- // Tombstones are not indexed and shouldn't be tested.
- if (!tree.itemStorage.contains(cacheAwareLink) && !cacheAwareLink.tombstone)
- errors.add(new IgniteException(cacheDataTreeEntryMissingError(name, cacheAwareLink)));
+ errors.add("Entry is missing in index[name=" + e.getKey() +
+ "cacheId=" + cacheAwareLink.cacheId +
+ ", partId=" + partId(pageId) +
+ ", pageIndex=" + pageIndex(pageId) +
+ ", itemId=" + itemId(cacheAwareLink.link) +
+ ", link=" + cacheAwareLink.link + ']');
}
if (errors.size() >= CHECK_PARTS_MAX_ERRORS_PER_PARTITION) {
- errors.add(new IgniteException("Too many errors (" + CHECK_PARTS_MAX_ERRORS_PER_PARTITION +
- ") found for partId=" + partId + ", stopping analysis for this partition."));
+ errors.add("Too many errors (" + CHECK_PARTS_MAX_ERRORS_PER_PARTITION +
+ ") found for partId=" + partId + ", stopping analysis for this partition.");
break;
}
@@ -604,1030 +601,661 @@ public class IgniteIndexReader implements AutoCloseable {
});
}
catch (IgniteCheckedException e) {
- errors.add(new IgniteException("Partition check failed, partId=" + i, e));
+ errors.add("Partition check failed, partId=" + partId);
}
if (!errors.isEmpty())
res.put(partId, errors);
- }
+ });
return res;
}
- /** */
- private String cacheDataTreeEntryMissingError(String treeName, CacheAwareLink cacheAwareLink) {
- long link = cacheAwareLink.link;
-
- long pageId = pageId(link);
-
- int itemId = itemId(link);
-
- int partId = partId(pageId);
+ /**
+ * Allocates buffer and does some work in closure, then frees the buffer.
+ *
+ * @param c Closure.
+ * @param <T> Result type.
+ * @return Result of closure.
+ * @throws IgniteCheckedException If failed.
+ */
+ private <T> T doWithBuffer(GridPlainClosure2<ByteBuffer, Long, T> c) throws IgniteCheckedException {
+ ByteBuffer buf = allocateBuffer(pageSize);
- int pageIdx = pageIndex(pageId);
+ try {
+ long addr = bufferAddress(buf);
- return "Entry is missing in index: " + treeName +
- ", cacheId=" + cacheAwareLink.cacheId + ", partId=" + partId +
- ", pageIndex=" + pageIdx + ", itemId=" + itemId + ", link=" + link;
+ return c.apply(buf, addr);
+ }
+ finally {
+ freeBuffer(buf);
+ }
}
- /**
- * @param partId Partition id.
- * @param flag Flag.
- * @return Id of partition meta page.
- */
- public static long partMetaPageId(int partId, byte flag) {
- return PageIdUtils.pageId(partId, flag, 0);
+ /** */
+ private void doWithoutErrors(RunnableX x, ScanContext ctx, long pageId) {
+ try {
+ x.run();
+ }
+ catch (Throwable e) {
+ ctx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e.getMessage());
+ }
}
/**
- * Compares result of traversals.
+ * Tries to get cache id and type id from index name.
*
- * @param treeInfos Traversal from root to leafs.
- * @param treeScans Traversal using horizontal scan.
+ * @param name Index name.
+ * @return Pair of cache id and type id.
*/
- private void compareTraversals(Map<String, TreeTraversalInfo> treeInfos, Map<String, TreeTraversalInfo> treeScans) {
- List<String> errors = new LinkedList<>();
-
- Set<String> treeIdxNames = new HashSet<>();
-
- treeInfos.forEach((name, tree) -> {
- treeIdxNames.add(name);
-
- TreeTraversalInfo scan = treeScans.get(name);
+ public IgnitePair<Integer> cacheAndTypeId(String name) {
+ return cacheTypeIds.computeIfAbsent(name, k -> {
+ Matcher mId = CACHE_TYPE_ID_SEARCH_PATTERN.matcher(k);
- if (scan == null) {
- errors.add("Tree was detected in " + RECURSIVE_TRAVERSE_NAME + " but absent in "
- + HORIZONTAL_SCAN_NAME + ": " + name);
+ if (mId.find())
+ return new IgnitePair<>(parseInt(mId.group("id")), parseInt(mId.group("typeId")));
- return;
- }
+ Matcher cId = CACHE_ID_SEARCH_PATTERN.matcher(k);
- if (tree.itemStorage.size() != scan.itemStorage.size())
- errors.add(compareError("items", name, tree.itemStorage.size(), scan.itemStorage.size(), null));
+ if (cId.find())
+ return new IgnitePair<>(parseInt(cId.group("id")), 0);
- Set<Class> classesInStat = new HashSet<>();
+ return new IgnitePair<>(0, 0);
+ });
+ }
- tree.ioStat.forEach((cls, cnt) -> {
- classesInStat.add(cls);
+ /** */
+ ScanContext createContext(int cacheId, FilePageStore store, ItemStorage items) {
+ return new ScanContext(cacheId, store, items);
+ }
- long scanCnt = scan.ioStat.getOrDefault(cls, 0L);
+ /** */
+ ProgressPrinter createProgressPrinter(String caption, long total) {
+ return new ProgressPrinter(System.out, caption, total);
+ }
- if (scanCnt != cnt)
- errors.add(compareError("pages", name, cnt, scanCnt, cls));
- });
+ /**
+ * @param rootPageId Root page id.
+ * @return File page store of given partition.
+ */
+ FilePageStore filePageStore(long rootPageId) {
+ int partId = partId(rootPageId);
- scan.ioStat.forEach((cls, cnt) -> {
- if (classesInStat.contains(cls))
- // Already checked.
- return;
+ return partId == INDEX_PARTITION ? idxStore : partStores[partId];
+ }
- errors.add(compareError("pages", name, 0, cnt, cls));
- });
- });
-
- treeScans.forEach((name, tree) -> {
- if (!treeIdxNames.contains(name))
- errors.add("Tree was detected in " + HORIZONTAL_SCAN_NAME + " but absent in "
- + RECURSIVE_TRAVERSE_NAME + ": " + name);
- });
-
- errors.forEach(this::printErr);
+ /**
+ * Reading a page from channel into buffer.
+ *
+ * @param buf Buffer.
+ * @param ch Source for reading pages.
+ * @param pageSize Size of page to read into buffer.
+ */
+ private boolean readNextPage(ByteBuffer buf, FileChannel ch, int pageSize) throws IOException {
+ assert buf.remaining() == pageSize;
- print("Comparing traversals detected " + errors.size() + " errors.");
- print("------------------");
+ do {
+ if (ch.read(buf) == -1)
+ break;
+ }
+ while (buf.hasRemaining());
+ if (!buf.hasRemaining() && PageIO.getPageId(buf) != 0)
+ return true; //pageSize bytes read && pageId != 0
+ else if (buf.remaining() == pageSize)
+ return false; //0 bytes read
+ else
+ // 1 <= readBytes < pageSize || readBytes == pagesIze && pageId != 0
+ throw new IgniteException("Corrupted page in partitionId " +
+ ", readByte=" + buf.position() + ", pageSize=" + pageSize);
}
/** */
- private String compareError(String itemName, String idxName, long fromRoot, long scan, Class pageType) {
- return format(
- "Different count of %s; index: %s, %s:%s, %s:%s" + (pageType == null ? "" : ", pageType: " + pageType.getName()),
- itemName,
- idxName,
- RECURSIVE_TRAVERSE_NAME,
- fromRoot,
- HORIZONTAL_SCAN_NAME,
- scan
- );
+ static long normalizePageId(long pageId) {
+ return pageId(partId(pageId), flag(pageId), pageIndex(pageId));
}
/**
- * Gets info about page lists.
+ * Reads pages into buffer.
*
- * @param metaPageListId Page list meta id.
- * @return Page list info.
+ * @param store Source for reading pages.
+ * @param pageId Page ID.
+ * @param buf Buffer.
*/
- private PageListsInfo getPageListsInfo(long metaPageListId) {
- Map<IgniteBiTuple<Long, Integer>, List<Long>> bucketsData = new HashMap<>();
-
- Set<Long> allPages = new HashSet<>();
-
- Map<Class<? extends PageIO>, Long> pageListStat = new HashMap<>();
-
- Map<Long, List<Throwable>> errors = new HashMap<>();
-
+ private <I extends PageIO> I readPage(FilePageStore store, long pageId, ByteBuffer buf) throws IgniteCheckedException {
try {
- doWithBuffer((buf, addr) -> {
- long nextMetaId = metaPageListId;
-
- while (nextMetaId != 0) {
- try {
- buf.rewind();
-
- readPage(idxStore, nextMetaId, buf);
-
- PagesListMetaIO io = PageIO.getPageIO(addr);
-
- Map<Integer, GridLongList> data = new HashMap<>();
-
- io.getBucketsData(addr, data);
-
- final long fNextMetaId = nextMetaId;
+ store.read(pageId, (ByteBuffer)buf.rewind(), false);
- data.forEach((k, v) -> {
- List<Long> listIds = LongStream.of(v.array()).map(IgniteIndexReader::normalizePageId).boxed().collect(toList());
-
- for (Long listId : listIds) {
- try {
- allPages.addAll(getPageList(listId, pageListStat));
- }
- catch (Exception e) {
- errors.put(listId, singletonList(e));
- }
- }
+ return PageIO.getPageIO(bufferAddress(buf));
+ }
+ catch (IgniteDataIntegrityViolationException | IllegalArgumentException e) {
+ // Replacing exception due to security reasons, as IgniteDataIntegrityViolationException prints page content.
+ // Catch IllegalArgumentException for output page information.
+ throw new IgniteException("Failed to read page, id=" + pageId + ", idx=" + pageIndex(pageId) +
+ ", file=" + store.getFileAbsolutePath());
+ }
+ }
- bucketsData.put(new IgniteBiTuple<>(fNextMetaId, k), listIds);
- });
+ /** */
+ protected <I extends PageIO> I readPage(ScanContext ctx, long pageId, ByteBuffer buf) throws IgniteCheckedException {
+ final I io = readPage(ctx.store, pageId, buf);
- nextMetaId = io.getNextMetaPageId(addr);
- }
- catch (Exception e) {
- errors.put(nextMetaId, singletonList(e));
+ ctx.onPageIO(io);
- nextMetaId = 0;
- }
- }
+ return io;
+ }
- return null;
- });
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
+ /**
+ * @return Tuple consisting of meta tree root page and pages list root page.
+ * @throws IgniteCheckedException If failed.
+ */
+ long[] partitionRoots(long metaPageId) throws IgniteCheckedException {
+ return doWithBuffer((buf, addr) -> {
+ PageMetaIO pageMetaIO = readPage(filePageStore(metaPageId), metaPageId, buf);
- return new PageListsInfo(bucketsData, allPages, pageListStat, errors);
+ return new long[] {
+ normalizePageId(pageMetaIO.getTreeRoot(addr)),
+ normalizePageId(pageMetaIO.getReuseListRoot(addr))
+ };
+ });
}
/**
- * Get single page list.
+ * Compares result of traversals.
*
- * @param pageListStartId Id of the start page of the page list.
- * @param pageStat Page types statistics.
- * @return List of page ids.
+ * @param recursiveScans Traversal from root to leafs.
+ * @param horizontalScans Traversal using horizontal scan.
*/
- private List<Long> getPageList(long pageListStartId, Map<Class<? extends PageIO>, Long> pageStat) {
- List<Long> res = new LinkedList<>();
-
- long nextNodeId = pageListStartId;
-
- ByteBuffer nodeBuf = allocateBuffer(pageSize);
- ByteBuffer pageBuf = allocateBuffer(pageSize);
-
- long nodeAddr = bufferAddress(nodeBuf);
- long pageAddr = bufferAddress(pageBuf);
-
- try {
- while (nextNodeId != 0) {
- try {
- nodeBuf.rewind();
+ private void compareScans(
+ Map<String, ScanContext> recursiveScans,
+ Map<String, ScanContext> horizontalScans
+ ) {
+ List<String> errors = new LinkedList<>();
- readPage(idxStore, nextNodeId, nodeBuf);
+ recursiveScans.forEach((name, rctx) -> {
+ ScanContext hctx = horizontalScans.get(name);
- PagesListNodeIO io = PageIO.getPageIO(nodeAddr);
+ if (hctx == null) {
+ errors.add("Tree was detected in " + RECURSIVE_TRAVERSE_NAME + " but absent in "
+ + HORIZONTAL_SCAN_NAME + ": " + name);
- for (int i = 0; i < io.getCount(nodeAddr); i++) {
- pageBuf.rewind();
+ return;
+ }
- long pageId = normalizePageId(io.getAt(nodeAddr, i));
+ if (rctx.items.size() != hctx.items.size())
+ errors.add(compareError("items", name, rctx.items.size(), hctx.items.size(), null));
- res.add(pageId);
+ rctx.ioStat.forEach((cls, cnt) -> {
+ long scanCnt = hctx.ioStat.getOrDefault(cls, 0L);
- readPage(idxStore, pageId, pageBuf);
+ if (scanCnt != cnt)
+ errors.add(compareError("pages", name, cnt, scanCnt, cls));
+ });
- PageIO pageIO = PageIO.getPageIO(pageAddr);
+ hctx.ioStat.forEach((cls, cnt) -> {
+ if (!rctx.ioStat.containsKey(cls))
+ errors.add(compareError("pages", name, 0, cnt, cls));
+ });
+ });
- pageStat.compute(pageIO.getClass(), (k, v) -> v == null ? 1 : v + 1);
- }
+ horizontalScans.forEach((name, hctx) -> {
+ if (!recursiveScans.containsKey(name))
+ errors.add("Tree was detected in " + HORIZONTAL_SCAN_NAME + " but absent in "
+ + RECURSIVE_TRAVERSE_NAME + ": " + name);
+ });
- nextNodeId = io.getNextId(nodeAddr);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e.getMessage(), e);
- }
- }
- }
- finally {
- freeBuffer(nodeBuf);
- freeBuffer(pageBuf);
- }
+ errors.forEach(log::severe);
- return res;
+ log.info("Comparing traversals detected " + errors.size() + " errors.");
+ log.info("------------------");
}
- /**
- * Traverse all trees in file and return their info.
- *
- * @param metaTreeRoot Meta tree root page id.
- * @return Index trees info.
- */
- private Map<String, TreeTraversalInfo> traverseAllTrees(
- String traverseProcCaption,
- long metaTreeRoot,
- Supplier<ItemStorage> itemStorageFactory,
- TraverseProc traverseProc
- ) {
- Map<String, TreeTraversalInfo> treeInfos = new LinkedHashMap<>();
+ /** Prints sequential file scan results. */
+ private void printSequentialScanInfo(ScanContext ctx) {
+ printIoStat("", "---- These pages types were encountered during sequential scan:", ctx.ioStat);
- TreeTraversalInfo metaTreeTraversalInfo =
- traverseProc.traverse(metaTreeRoot, META_TREE_NAME, new ItemsListStorage<IndexStorageImpl.IndexItem>());
+ if (!ctx.errors.isEmpty()) {
+ log.severe("----");
+ log.severe("Errors:");
- treeInfos.put(META_TREE_NAME, metaTreeTraversalInfo);
+ ctx.errors.values().forEach(e -> log.severe(e.get(0)));
+ }
- ProgressPrinter progressPrinter =
- new ProgressPrinter(System.out, traverseProcCaption, metaTreeTraversalInfo.itemStorage.size());
+ log.info("----");
- metaTreeTraversalInfo.itemStorage.forEach(item -> {
- progressPrinter.printProgress();
+ SystemViewCommand.printTable(
+ null,
+ Arrays.asList(STRING, NUMBER),
+ Arrays.asList(
+ Arrays.asList("Total pages encountered during sequential scan:", ctx.ioStat.values().stream().mapToLong(a -> a).sum()),
+ Arrays.asList("Total errors occurred during sequential scan: ", ctx.errors.size())
+ ),
+ log
+ );
- IndexStorageImpl.IndexItem idxItem = (IndexStorageImpl.IndexItem)item;
+ if (idxFilter != null)
+ log.info("Orphan pages were not reported due to --indexes filter.");
- if (nonNull(idxFilter) && !idxFilter.test(idxItem.nameString()))
- return;
+ log.info("Note that some pages can be occupied by meta info, tracking info, etc., so total page count can differ " +
+ "from count of pages found in index trees and page lists.");
+ }
- TreeTraversalInfo treeTraversalInfo =
- traverseProc.traverse(normalizePageId(idxItem.pageId()), idxItem.nameString(), itemStorageFactory.get());
+ /** Print check partitions info. */
+ private void printCheckPartsInfo(Map<Integer, List<String>> checkPartsErrors) {
+ log.info("");
- treeInfos.put(idxItem.toString(), treeTraversalInfo);
- });
+ printErrors("",
+ "Partitions check:",
+ "Partitions check detected no errors.",
+ "Errors detected in partition, partId=%s",
+ checkPartsErrors
+ );
- return treeInfos;
+ log.info("Partition check finished, total errors: " +
+ checkPartsErrors.values().stream().mapToInt(List::size).sum() + ", total problem partitions: " +
+ checkPartsErrors.size()
+ );
}
- /**
- * Prints traversal info.
- *
- * @param treeInfos Tree traversal info.
- */
- private void printTraversalResults(String prefix, Map<String, TreeTraversalInfo> treeInfos) {
- print("\n" + prefix + "Tree traversal results");
+ /** Prints traversal info. */
+ private void printScanResults(String prefix, Map<String, ScanContext> ctxs) {
+ log.info(prefix + "Tree traversal results");
- Map<Class<? extends PageIO>, Long> totalStat = new HashMap<>();
+ Map<Class<? extends PageIO>, Long> ioStat = new HashMap<>();
- AtomicInteger totalErr = new AtomicInteger(0);
+ int totalErr = 0;
// Map (cacheId, typeId) -> (map idxName -> size))
Map<IgnitePair<Integer>, Map<String, Long>> cacheIdxSizes = new HashMap<>();
- treeInfos.forEach((idxName, validationInfo) -> {
- print(prefix + "-----");
- print(prefix + "Index tree: " + idxName);
- print(prefix + "-- Page stat:");
+ for (Map.Entry<String, ScanContext> e : ctxs.entrySet()) {
+ String idxName = e.getKey();
+ ScanContext ctx = e.getValue();
- validationInfo.ioStat.forEach((cls, cnt) -> {
- print(prefix + cls.getSimpleName() + ": " + cnt);
+ log.info(prefix + "-----");
+ log.info(prefix + "Index tree: " + idxName);
+ printIoStat(prefix, "---- Page stat:", ctx.ioStat);
- totalStat.compute(cls, (k, v) -> v == null ? 1 : v + 1);
- });
+ ctx.ioStat.forEach((cls, cnt) -> ScanContext.onPageIO(cls, ioStat, cnt));
- print(prefix + "-- Count of items found in leaf pages: " + validationInfo.itemStorage.size());
+ log.info(prefix + "---- Count of items found in leaf pages: " + ctx.items.size());
printErrors(
prefix,
"Errors:",
"No errors occurred while traversing.",
"Page id=%s, exceptions:",
- true,
- validationInfo.errors
+ ctx.errors
);
- totalErr.addAndGet(validationInfo.errors.size());
+ totalErr += ctx.errors.size();
- cacheIdxSizes.computeIfAbsent(getCacheAndTypeId(idxName), k -> new HashMap<>())
- .put(idxName, validationInfo.itemStorage.size());
- });
+ cacheIdxSizes
+ .computeIfAbsent(cacheAndTypeId(idxName), k -> new HashMap<>())
+ .put(idxName, ctx.items.size());
+ }
+
+ log.info(prefix + "----");
- print(prefix + "---");
+ printIoStat(prefix, "Total page stat collected during trees traversal:", ioStat);
- printPageStat(prefix, "Total page stat collected during trees traversal:", totalStat);
+ log.info("");
- print("");
+ boolean sizeConsistencyErrorsFound = false;
- AtomicBoolean sizeConsistencyErrorsFound = new AtomicBoolean(false);
+ for (Map.Entry<IgnitePair<Integer>, Map<String, Long>> entry : cacheIdxSizes.entrySet()) {
+ IgnitePair<Integer> cacheTypeId = entry.getKey();
+ Map<String, Long> idxSizes = entry.getValue();
- cacheIdxSizes.forEach((cacheTypeId, idxSizes) -> {
if (idxSizes.values().stream().distinct().count() > 1) {
- sizeConsistencyErrorsFound.set(true);
+ sizeConsistencyErrorsFound = true;
- totalErr.incrementAndGet();
+ totalErr++;
- printErr("Index size inconsistency: cacheId=" + cacheTypeId.get1() + ", typeId=" + cacheTypeId.get2());
+ log.severe("Index size inconsistency: cacheId=" + cacheTypeId.get1() + ", typeId=" + cacheTypeId.get2());
- idxSizes.forEach((name, size) -> printErr(" Index name: " + name + ", size=" + size));
+ idxSizes.forEach((name, size) -> log.severe(" Index name: " + name + ", size=" + size));
}
- });
+ }
- if (!sizeConsistencyErrorsFound.get())
- print(prefix + "No index size consistency errors found.");
+ if (!sizeConsistencyErrorsFound)
+ log.info(prefix + "No index size consistency errors found.");
- print("");
- print(prefix + "Total trees: " + treeInfos.keySet().size());
- print(prefix + "Total pages found in trees: " + totalStat.values().stream().mapToLong(a -> a).sum());
- print(prefix + "Total errors during trees traversal: " + totalErr.get());
- print("");
+ log.info("");
- print("------------------");
- }
+ SystemViewCommand.printTable(
+ null,
+ Arrays.asList(STRING, NUMBER),
+ Arrays.asList(
+ Arrays.asList(prefix + "Total trees: ", ctxs.keySet().size()),
+ Arrays.asList(prefix + "Total pages found in trees: ", ioStat.values().stream().mapToLong(a -> a).sum()),
+ Arrays.asList(prefix + "Total errors during trees traversal: ", totalErr)
+ ),
+ log
+ );
- /**
- * Tries to get cache id from index tree name.
- *
- * @param name Index name.
- * @return Cache id.
- */
- public static int getCacheId(String name) {
- return getCacheAndTypeId(name).get1();
+ log.info("");
+ log.info("------------------");
}
/**
- * Tries to get cache id and type id from index tree name.
+ * Prints page lists info.
*
- * @param name Index name.
- * @return Pair of cache id and type id.
+ * @param reuseListRoot Page id.
*/
- public static IgnitePair<Integer> getCacheAndTypeId(String name) {
- return CACHE_TYPE_IDS.computeIfAbsent(name, k -> {
- Matcher mId = CACHE_TYPE_ID_SEARCH_PATTERN.matcher(k);
+ private void printPagesListsInfo(long reuseListRoot) throws IgniteCheckedException {
+ if (reuseListRoot == 0) {
+ log.severe("No page lists meta info found.");
- if (mId.find()) {
- String id = mId.group("id");
-
- String typeId = mId.group("typeId");
-
- return new IgnitePair<>(parseInt(id), parseInt(typeId));
- }
- else {
- Matcher cId = CACHE_ID_SEARCH_PATTERN.matcher(k);
-
- if (cId.find()) {
- String id = cId.group("id");
+ return;
+ }
- return new IgnitePair<>(parseInt(id), 0);
- }
- }
+ PageListsInfo pageListsInfo = pageListsInfo(reuseListRoot);
- return new IgnitePair<>(0, 0);
- });
- }
-
- /**
- * Prints page lists info.
- *
- * @param pageListsInfo Page lists info.
- */
- private void printPagesListsInfo(PageListsInfo pageListsInfo) {
- String prefix = PAGE_LISTS_PREFIX;
-
- print("\n" + prefix + "Page lists info.");
+ log.info(PAGE_LISTS_PREFIX + "Page lists info.");
if (!pageListsInfo.bucketsData.isEmpty())
- print(prefix + "---Printing buckets data:");
+ log.info(PAGE_LISTS_PREFIX + "---- Printing buckets data:");
pageListsInfo.bucketsData.forEach((bucket, bucketData) -> {
- GridStringBuilder sb = new GridStringBuilder(prefix)
+ GridStringBuilder sb = new GridStringBuilder(PAGE_LISTS_PREFIX)
.a("List meta id=")
.a(bucket.get1())
.a(", bucket number=")
.a(bucket.get2())
.a(", lists=[")
- .a(bucketData.stream().map(String::valueOf).collect(joining(", ")))
+ .a(bucketData.stream().map(IgniteIndexReader::normalizePageId).map(String::valueOf).collect(joining(", ")))
.a("]");
- print(sb.toString());
+ log.info(sb.toString());
});
- printPageStat(prefix, "-- Page stat:", pageListsInfo.pageListStat);
-
- printErrors(prefix, "---Errors:", "---No errors.", "Page id: %s, exception: ", true, pageListsInfo.errors);
-
- print("");
- print(prefix + "Total index pages found in lists: " + pageListsInfo.allPages.size());
- print(prefix + "Total errors during lists scan: " + pageListsInfo.errors.size());
- print("------------------");
- }
-
- /**
- * Traverse single index tree from root to leafs.
- *
- * @param rootPageId Root page id.
- * @param treeName Tree name.
- * @param innerCb Inner pages callback.
- * @param leafCb Leaf pages callback.
- * @param itemCb Items callback.
- * @param itemStorage Items storage.
- * @return Tree traversal info.
- */
- TreeTraversalInfo traverseTree(
- long rootPageId,
- String treeName,
- @Nullable PageCallback innerCb,
- @Nullable PageCallback leafCb,
- @Nullable ItemCallback itemCb,
- ItemStorage itemStorage
- ) {
- Set<Long> innerPageIds = new HashSet<>();
-
- PageCallback innerCb0 = (content, pageId) -> {
- if (innerCb != null)
- innerCb.cb(content, pageId);
+ printIoStat(PAGE_LISTS_PREFIX, "---- Page stat:", pageListsInfo.ioStat);
- innerPageIds.add(normalizePageId(pageId));
- };
+ printErrors(PAGE_LISTS_PREFIX, "---- Errors:", "---- No errors.", "Page id: %s, exception: ", pageListsInfo.errors);
- ItemCallback itemCb0 = (currPageId, item, link) -> {
- if (itemCb != null)
- itemCb.cb(currPageId, item, link);
+ log.info("");
- itemStorage.add(item);
- };
-
- TreeTraverseContext ctx = new TreeTraverseContext(
- treeName,
- filePageStore(partId(rootPageId)),
- innerCb0,
- leafCb,
- itemCb0
- );
-
- getTreeNode(rootPageId, ctx);
-
- return new TreeTraversalInfo(ctx.ioStat, ctx.errors, innerPageIds, rootPageId, itemStorage);
- }
-
- /**
- * Traverse single index tree from root to leafs.
- *
- * @param rootPageId Root page id.
- * @param treeName Tree name.
- * @param itemStorage Items storage.
- * @return Tree traversal info.
- */
- TreeTraversalInfo traverseTree(long rootPageId, String treeName, ItemStorage itemStorage) {
- return traverseTree(rootPageId, treeName, null, null, null, itemStorage);
- }
-
- /**
- * Traverse single index tree by each level horizontally.
- *
- * @param rootPageId Root page id.
- * @param treeName Tree name.
- * @param itemStorage Items storage.
- * @return Tree traversal info.
- */
- private TreeTraversalInfo horizontalTreeScan(
- long rootPageId,
- String treeName,
- ItemStorage itemStorage
- ) {
- TreeTraverseContext treeCtx = new TreeTraverseContext(
- treeName,
- filePageStore(partId(rootPageId)),
- null,
+ SystemViewCommand.printTable(
null,
- null
+ Arrays.asList(STRING, NUMBER),
+ Arrays.asList(
+ Arrays.asList(PAGE_LISTS_PREFIX + "Total index pages found in lists:", pageListsInfo.pagesCnt),
+ Arrays.asList(PAGE_LISTS_PREFIX + "Total errors during lists scan:", pageListsInfo.errors.size())
+ ),
+ log
);
- ByteBuffer buf = allocateBuffer(pageSize);
-
- try {
- long addr = bufferAddress(buf);
-
- readPage(treeCtx.store, rootPageId, buf);
-
- PageIO pageIO = PageIO.getPageIO(addr);
-
- if (!(pageIO instanceof BPlusMetaIO))
- throw new IgniteException("Root page is not meta, pageId=" + rootPageId);
-
- BPlusMetaIO metaIO = (BPlusMetaIO)pageIO;
-
- treeCtx.ioStat.compute(metaIO.getClass(), (k, v) -> v == null ? 1 : v + 1);
-
- int lvlsCnt = metaIO.getLevelsCount(addr);
-
- long[] firstPageIds = IntStream.range(0, lvlsCnt).mapToLong(i -> metaIO.getFirstPageId(addr, i)).toArray();
-
- for (int i = 0; i < lvlsCnt; i++) {
- long pageId = firstPageIds[i];
-
- while (pageId > 0) {
- try {
- buf.rewind();
-
- readPage(treeCtx.store, pageId, buf);
-
- pageIO = PageIO.getPageIO(addr);
-
- if (i == 0 && !(pageIO instanceof BPlusLeafIO))
- throw new IgniteException("Not-leaf page found on leaf level, pageId=" + pageId + ", level=" + i);
-
- if (!(pageIO instanceof BPlusIO))
- throw new IgniteException("Not-BPlus page found, pageId=" + pageId + ", level=" + i);
-
- treeCtx.ioStat.compute(pageIO.getClass(), (k, v) -> v == null ? 1 : v + 1);
-
- if (pageIO instanceof BPlusLeafIO) {
- PageIOProcessor ioProcessor = getIOProcessor(pageIO);
-
- PageContent pageContent = ioProcessor.getContent(pageIO, addr, pageId, treeCtx);
-
- pageContent.items.forEach(itemStorage::add);
- }
-
- pageId = ((BPlusIO<?>)pageIO).getForward(addr);
- }
- catch (Throwable e) {
- treeCtx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e);
-
- pageId = 0;
- }
- }
- }
- }
- catch (Throwable e) {
- treeCtx.errors.computeIfAbsent(rootPageId, k -> new LinkedList<>()).add(e);
- }
- finally {
- freeBuffer(buf);
- }
-
- return new TreeTraversalInfo(treeCtx.ioStat, treeCtx.errors, null, rootPageId, itemStorage);
- }
-
- /**
- * @param partId Partition id.
- * @return File page store of given partition.
- */
- private FilePageStore filePageStore(int partId) {
- return partId == INDEX_PARTITION ? idxStore : partStores[partId];
- }
-
- /**
- * Gets tree node and all its children.
- *
- * @param pageId Page id, where tree node is located.
- * @param nodeCtx Tree traverse context.
- * @return Tree node.
- */
- private TreeNode getTreeNode(long pageId, TreeTraverseContext nodeCtx) {
- PageContent pageContent;
-
- PageIOProcessor ioProcessor;
-
- try {
- final ByteBuffer buf = allocateBuffer(pageSize);
-
- try {
- readPage(nodeCtx.store, pageId, buf);
-
- final long addr = bufferAddress(buf);
-
- final PageIO io = PageIO.getPageIO(addr);
-
- nodeCtx.ioStat.compute(io.getClass(), (k, v) -> v == null ? 1 : v + 1);
-
- ioProcessor = getIOProcessor(io);
-
- pageContent = ioProcessor.getContent(io, addr, pageId, nodeCtx);
- }
- finally {
- freeBuffer(buf);
- }
-
- return ioProcessor.getNode(pageContent, pageId, nodeCtx);
- }
- catch (Throwable e) {
- nodeCtx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e);
-
- return new TreeNode(pageId, null, "exception: " + e.getMessage(), Collections.emptyList());
- }
+ log.info("------------------");
}
/** */
- private PageIOProcessor getIOProcessor(PageIO io) {
- if (io instanceof BPlusLeafIO)
- return leafPageIOProcessor;
- else if (io instanceof BPlusInnerIO)
- return innerPageIOProcessor;
- else if (io instanceof BPlusMetaIO)
- return metaPageIOProcessor;
- else
- return null;
+ private String compareError(String itemName, String idxName, long fromRoot, long scan, Class<? extends PageIO> io) {
+ return format(
+ "Different count of %s; index: %s, %s:%s, %s:%s" + (io == null ? "" : ", pageType: " + io.getSimpleName()),
+ itemName,
+ idxName,
+ RECURSIVE_TRAVERSE_NAME,
+ fromRoot,
+ HORIZONTAL_SCAN_NAME,
+ scan
+ );
}
- /** {@inheritDoc} */
- @Override public void close() throws StorageException {
- if (nonNull(idxStore))
- idxStore.stop(false);
+ /** */
+ private void printErrors(String prefix, String caption, String emptyCaption, String msg, Map<?, List<String>> errors) {
+ if (errors.isEmpty()) {
+ log.info(prefix + emptyCaption);
- if (nonNull(partStores)) {
- for (FilePageStore store : partStores) {
- if (nonNull(store))
- store.stop(false);
- }
+ return;
}
- }
- /**
- * Reading a page from channel into buffer.
- *
- * @param buf Buffer.
- * @param ch Source for reading pages.
- * @param pageSize Size of page to read into buffer.
- */
- private boolean readNextPage(ByteBuffer buf, FileChannel ch, int pageSize) throws IOException {
- assert buf.remaining() == pageSize;
+ log.info(prefix + ERROR_PREFIX + caption);
- do {
- if (ch.read(buf) == -1)
- break;
- }
- while (buf.hasRemaining());
+ errors.forEach((k, v) -> {
+ log.info(prefix + ERROR_PREFIX + format(msg, k.toString()));
- if (!buf.hasRemaining() && PageIO.getPageId(buf) != 0)
- return true; //pageSize bytes read && pageId != 0
- else if (buf.remaining() == pageSize)
- return false; //0 bytes read
- else
- // 1 <= readBytes < pageSize || readBytes == pagesIze && pageId != 0
- throw new IgniteException("Corrupted page in partitionId " +
- ", readByte=" + buf.position() + ", pageSize=" + pageSize);
+ v.forEach(log::severe);
+ });
}
- /**
- * Entry point.
- *
- * @param args Arguments.
- * @throws Exception If failed.
- */
- public static void main(String[] args) throws Exception {
- System.out.println("THIS UTILITY MUST BE LAUNCHED ON PERSISTENT STORE WHICH IS NOT UNDER RUNNING GRID!");
-
- List<CLIArgument<?>> argsConfiguration = asList(
- mandatoryArg(
- DIR.arg(),
- "partition directory, where " + INDEX_FILE_NAME + " and (optionally) partition files are located.",
- String.class
- ),
- optionalArg(PART_CNT.arg(), "full partitions count in cache group.", Integer.class, () -> 0),
- optionalArg(PAGE_SIZE.arg(), "page size.", Integer.class, () -> 4096),
- optionalArg(PAGE_STORE_VER.arg(), "page store version.", Integer.class, () -> 2),
- optionalArg(INDEXES.arg(), "you can specify index tree names that will be processed, separated by comma " +
- "without spaces, other index trees will be skipped.", String[].class, () -> null),
- optionalArg(DEST_FILE.arg(),
- "file to print the report to (by default report is printed to console).", String.class, () -> null),
- optionalArg(CHECK_PARTS.arg(),
- "check cache data tree in partition files and it's consistency with indexes.", Boolean.class, () -> false)
- );
-
- CLIArgumentParser p = new CLIArgumentParser(argsConfiguration);
-
- if (args.length == 0) {
- System.out.println(p.usage());
+ /** */
+ private void printIoStat(String prefix, String caption, Map<Class<? extends PageIO>, Long> ioStat) {
+ if (caption != null)
+ log.info(prefix + caption + (ioStat.isEmpty() ? " empty" : ""));
+ if (ioStat.isEmpty())
return;
- }
-
- p.parse(asList(args).iterator());
-
- String destFile = p.get(DEST_FILE.arg());
- OutputStream destStream = isNull(destFile) ? null : new FileOutputStream(destFile);
+ List<List<?>> data = new ArrayList<>(ioStat.size());
- String dir = p.get(DIR.arg());
+ ioStat.forEach((cls, cnt) -> data.add(Arrays.asList(prefix + cls.getSimpleName(), cnt)));
- int pageSize = p.get(PAGE_SIZE.arg());
-
- IgniteIndexReaderFilePageStoreFactory filePageStoreFactory = new IgniteIndexReaderFilePageStoreFactoryImpl(
- new File(dir),
- pageSize,
- p.get(PART_CNT.arg()),
- p.get(PAGE_STORE_VER.arg())
+ SystemViewCommand.printTable(
+ null,
+ Arrays.asList(STRING, NUMBER),
+ data,
+ log
);
-
- String[] idxArr = p.get(INDEXES.arg());
- Set<String> idxSet = isNull(idxArr) ? null : new HashSet<>(asList(idxArr));
-
- try (IgniteIndexReader reader = new IgniteIndexReader(
- isNull(idxSet) ? null : idxSet::contains,
- p.get(CHECK_PARTS.arg()),
- isNull(destStream) ? null : new PrintStream(destFile),
- filePageStoreFactory
- )) {
- reader.readIdx();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(INDEX_FILE_NAME + " scan problem", e);
- }
}
- /**
- * Enum of possible utility arguments.
- */
- public enum Args {
- /** */
- DIR("--dir"),
- /** */
- PART_CNT("--part-cnt"),
- /** */
- PAGE_SIZE("--page-size"),
- /** */
- PAGE_STORE_VER("--page-store-ver"),
- /** */
- INDEXES("--indexes"),
- /** */
- DEST_FILE("--dest-file"),
- /** */
- CHECK_PARTS("--check-parts");
-
- /** */
- private final String arg;
-
- /** */
- Args(String arg) {
- this.arg = arg;
- }
+ /** {@inheritDoc} */
+ @Override public void close() throws StorageException {
+ idxStore.stop(false);
- /** */
- public String arg() {
- return arg;
+ for (FilePageStore store : partStores) {
+ if (nonNull(store))
+ store.stop(false);
}
}
- /**
- *
- */
- private interface BufferClosure<T> {
+ /** */
+ private interface Scanner {
/** */
- T apply(ByteBuffer buf, Long addr) throws IgniteCheckedException;
+ ScanContext scan(long rootId, String idx, ItemStorage items);
}
- /**
- *
- */
- private interface TraverseProc {
+ /** Processor for page IOs. */
+ private abstract class TreePageVisitor {
/** */
- TreeTraversalInfo traverse(long rootId, String treeName, ItemStorage itemStorage);
- }
+ protected void readAndVisit(long pageId, ScanContext ctx) {
+ doWithoutErrors(() -> doWithBuffer((buf, addr) -> {
+ final PageIO io = readPage(ctx, pageId, buf);
+
+ if (io instanceof BPlusLeafIO)
+ leafPageVisitor.visit(addr, ctx);
+ else if (io instanceof BPlusInnerIO)
+ innerPageVisitor.visit(addr, ctx);
+ else
+ throw new IllegalArgumentException("Unknown io [io=" + io.getClass().getSimpleName() + ']');
- /**
- * Processor for page IOs.
- */
- private interface PageIOProcessor {
- /**
- * Gets deserialized content.
- * @param io Page IO.
- * @param addr Page address.
- * @param pageId Page id.
- * @param nodeCtx Tree traversal context.
- * @return Page content.
- */
- PageContent getContent(PageIO io, long addr, long pageId, TreeTraverseContext nodeCtx);
-
- /**
- * Gets node info from page contents.
- * @param content Page content.
- * @param pageId Page id.
- * @param nodeCtx Tree traversal context.
- * @return Tree node info.
- */
- TreeNode getNode(PageContent content, long pageId, TreeTraverseContext nodeCtx);
+ return null;
+ }), ctx, pageId);
+ }
}
- /**
- *
- */
- private class MetaPageIOProcessor implements PageIOProcessor {
+ /** */
+ private class MetaPageVisitor extends TreePageVisitor {
/** {@inheritDoc} */
- @Override public PageContent getContent(PageIO io, long addr, long pageId, TreeTraverseContext nodeCtx) {
- BPlusMetaIO bPlusMetaIO = (BPlusMetaIO)io;
-
- int rootLvl = bPlusMetaIO.getRootLevel(addr);
- long rootId = bPlusMetaIO.getFirstPageId(addr, rootLvl);
+ @Override protected void readAndVisit(long pageId, ScanContext ctx) {
+ doWithoutErrors(() -> {
+ long firstPageId = doWithBuffer((buf, addr) -> {
+ BPlusMetaIO io = readPage(ctx, pageId, buf);
- return new PageContent(io, singletonList(rootId), null, null);
- }
+ return io.getFirstPageId(addr, io.getRootLevel(addr));
+ });
- /** {@inheritDoc} */
- @Override public TreeNode getNode(PageContent content, long pageId, TreeTraverseContext nodeCtx) {
- return new TreeNode(pageId, content.io, null, singletonList(getTreeNode(content.linkedPageIds.get(0), nodeCtx)));
+ super.readAndVisit(firstPageId, ctx);
+ }, ctx, pageId);
}
}
- /**
- *
- */
- private class InnerPageIOProcessor implements PageIOProcessor {
+ /** */
+ private class LevelsPageVisitor extends TreePageVisitor {
/** {@inheritDoc} */
- @Override public PageContent getContent(PageIO io, long addr, long pageId, TreeTraverseContext nodeCtx) {
- BPlusInnerIO<?> innerIo = (BPlusInnerIO<?>)io;
+ @Override protected void readAndVisit(long rootPageId, ScanContext ctx) {
+ doWithoutErrors(() -> doWithBuffer((buf, addr) -> {
+ PageIO pageIO = readPage(ctx, rootPageId, buf);
- int cnt = innerIo.getCount(addr);
+ if (!(pageIO instanceof BPlusMetaIO))
+ throw new IgniteException("Root page is not meta, pageId=" + rootPageId);
- List<Long> childrenIds;
+ BPlusMetaIO metaIO = (BPlusMetaIO)pageIO;
- if (cnt > 0) {
- childrenIds = new ArrayList<>(cnt + 1);
+ int lvlsCnt = metaIO.getLevelsCount(addr);
- for (int i = 0; i < cnt; i++)
- childrenIds.add(innerIo.getLeft(addr, i));
+ long[] firstPageIds = IntStream.range(0, lvlsCnt).mapToLong(i -> metaIO.getFirstPageId(addr, i)).toArray();
- childrenIds.add(innerIo.getRight(addr, cnt - 1));
- }
- else {
- long left = innerIo.getLeft(addr, 0);
+ for (int i = 0; i < lvlsCnt; i++) {
+ long pageId = firstPageIds[i];
- childrenIds = left == 0 ? Collections.<Long>emptyList() : singletonList(left);
- }
+ try {
+ while (pageId > 0) {
+ pageIO = readPage(ctx, pageId, buf);
- return new PageContent(io, childrenIds, null, null);
- }
+ if (i == 0 && !(pageIO instanceof BPlusLeafIO))
+ throw new IgniteException("Not-leaf page found on leaf level [pageId=" + pageId + ", level=0]");
- /** {@inheritDoc} */
- @Override public TreeNode getNode(PageContent content, long pageId, TreeTraverseContext nodeCtx) {
- List<TreeNode> children = new ArrayList<>(content.linkedPageIds.size());
+ if (!(pageIO instanceof BPlusIO))
+ throw new IgniteException("Not-BPlus page found [pageId=" + pageId + ", level=" + i + ']');
- for (Long id : content.linkedPageIds)
- children.add(getTreeNode(id, nodeCtx));
+ if (pageIO instanceof BPlusLeafIO)
+ leafPageVisitor.visit(addr, ctx);
- if (nodeCtx.innerCb != null)
- nodeCtx.innerCb.cb(content, pageId);
+ pageId = ((BPlusIO<?>)pageIO).getForward(addr);
+ }
+ }
+ catch (Throwable e) {
+ ctx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e.getMessage());
+ }
+ }
- return new TreeNode(pageId, content.io, null, children);
+ return null;
+ }), ctx, rootPageId);
}
}
- /**
- *
- */
- private class LeafPageIOProcessor implements PageIOProcessor {
- /** {@inheritDoc} */
- @Override public PageContent getContent(PageIO io, long addr, long pageId, TreeTraverseContext nodeCtx) {
- GridStringBuilder sb = new GridStringBuilder();
-
- List<Object> items = new LinkedList<>();
-
- BPlusLeafIO<?> leafIO = (BPlusLeafIO<?>)io;
-
- for (int j = 0; j < leafIO.getCount(addr); j++) {
- Object idxItem = null;
-
- try {
- if (io instanceof IndexStorageImpl.MetaStoreLeafIO) {
- idxItem = ((BPlusIO<IndexStorageImpl.IndexItem>)io).getLookupRow(null, addr, j);
-
- sb.a(idxItem + " ");
- }
- else
- idxItem = getLeafItem(leafIO, pageId, addr, j, nodeCtx);
- }
- catch (Exception e) {
- nodeCtx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e);
- }
+ /** */
+ private class InnerPageVisitor extends TreePageVisitor {
+ /** */
+ public void visit(long addr, ScanContext ctx) throws IgniteCheckedException {
+ BPlusInnerIO<?> io = PageIO.getPageIO(addr);
- if (idxItem != null)
- items.add(idxItem);
- }
+ pageIds.add(normalizePageId(PageIO.getPageId(addr)));
- return new PageContent(io, null, items, sb.toString());
+ for (long id : children(io, addr))
+ readAndVisit(id, ctx);
}
/** */
- private Object getLeafItem(BPlusLeafIO<?> io, long pageId, long addr, int idx, TreeTraverseContext nodeCtx) {
- if (isLinkIo(io)) {
- final long link = getLink(io, addr, idx);
+ private long[] children(BPlusInnerIO<?> io, long addr) {
+ int cnt = io.getCount(addr);
- final int cacheId;
+ if (cnt == 0) {
+ long left = io.getLeft(addr, 0);
- if (io instanceof AbstractDataLeafIO && ((AbstractDataLeafIO)io).storeCacheId())
- cacheId = ((RowLinkIO)io).getCacheId(addr, idx);
- else
- cacheId = nodeCtx.cacheId;
+ return left == 0 ? U.EMPTY_LONGS : new long[] {left};
+ }
- boolean tombstone = false;
+ long[] children = new long[cnt + 1];
- if (partCnt > 0) {
- try {
- long linkedPageId = pageId(link);
+ for (int i = 0; i < cnt; i++)
+ children[i] = io.getLeft(addr, i);
- int linkedPagePartId = partId(linkedPageId);
+ children[cnt] = io.getRight(addr, cnt - 1);
- if (missingPartitions.contains(linkedPagePartId))
- return new CacheAwareLink(cacheId, link, false); // just skip
+ return children;
+ }
+ }
- int linkedItemId = itemId(link);
+ /** */
+ private class LeafPageVisitor extends TreePageVisitor {
+ /** */
+ public void visit(long addr, ScanContext ctx) throws IgniteCheckedException {
+ List<Object> items = new LinkedList<>();
- if (linkedPagePartId > partStores.length - 1) {
- missingPartitions.add(linkedPagePartId);
+ BPlusLeafIO<?> io = PageIO.getPageIO(addr);
- throw new IgniteException("Calculated data page partition id exceeds given partitions " +
- "count: " + linkedPagePartId + ", partCnt=" + partCnt);
- }
+ doWithoutErrors(() -> {
+ for (int i = 0; i < io.getCount(addr); i++) {
+ if (io instanceof IndexStorageImpl.MetaStoreLeafIO)
+ items.add(((BPlusIO<IndexStorageImpl.IndexItem>)io).getLookupRow(null, addr, i));
+ else
+ items.add(leafItem(io, addr, i, ctx));
+ }
+ }, ctx, PageIO.getPageId(addr));
- final FilePageStore store = partStores[linkedPagePartId];
+ ctx.onLeafPage(PageIO.getPageId(addr), items);
+ }
- if (store == null) {
- missingPartitions.add(linkedPagePartId);
+ /** */
+ private Object leafItem(BPlusLeafIO<?> io, long addr, int idx, ScanContext ctx) {
+ if (!(io instanceof InlineIO || io instanceof PendingRowIO || io instanceof RowLinkIO))
+ throw new IgniteException("Unexpected page io: " + io.getClass().getSimpleName());
- throw new IgniteException("Corresponding store wasn't found for partId=" +
- linkedPagePartId + ". Does partition file exist?");
- }
+ final long link = link(io, addr, idx);
- tombstone = doWithBuffer((dataBuf, dataBufAddr) -> {
- readPage(store, linkedPageId, dataBuf);
+ final int cacheId = (io instanceof AbstractDataLeafIO && ((AbstractDataLeafIO)io).storeCacheId())
+ ? ((RowLinkIO)io).getCacheId(addr, idx)
+ : ctx.cacheId;
- PageIO dataIo = PageIO.getPageIO(getType(dataBuf), getVersion(dataBuf));
+ if (partCnt == 0)
+ return new CacheAwareLink(cacheId, link);
- if (dataIo instanceof AbstractDataPageIO) {
- AbstractDataPageIO<?> dataPageIO = (AbstractDataPageIO<?>)dataIo;
+ long linkedPageId = pageId(link);
- DataPagePayload payload = dataPageIO.readPayload(dataBufAddr, linkedItemId, pageSize);
+ int linkedPagePartId = partId(linkedPageId);
- if (payload.offset() <= 0 || payload.payloadSize() <= 0) {
- GridStringBuilder payloadInfo = new GridStringBuilder("Invalid data page payload: ")
- .a("off=").a(payload.offset())
- .a(", size=").a(payload.payloadSize())
- .a(", nextLink=").a(payload.nextLink());
+ if (missingPartitions.contains(linkedPagePartId))
+ return new CacheAwareLink(cacheId, link); // just skip
- throw new IgniteException(payloadInfo.toString());
- }
+ doWithoutErrors(() -> {
+ if (linkedPagePartId > partStores.length - 1) {
+ missingPartitions.add(linkedPagePartId);
- if (payload.nextLink() == 0) {
- if (io instanceof MvccDataLeafIO)
- return false;
+ throw new IgniteException("Calculated data page partition id exceeds given partitions " +
+ "count: " + linkedPagePartId + ", partCnt=" + partCnt);
+ }
- int off = payload.offset();
+ if (partStores[linkedPagePartId] == null) {
+ missingPartitions.add(linkedPagePartId);
- int len = PageUtils.getInt(dataBufAddr, off);
+ throw new IgniteException("Corresponding store wasn't found for partId=" +
+ linkedPagePartId + ". Does partition file exist?");
+ }
- byte type = PageUtils.getByte(dataBufAddr, off + len + 9);
+ doWithBuffer((dataBuf, dataBufAddr) -> {
+ PageIO dataIo = readPage(partStores[linkedPagePartId], linkedPageId, dataBuf);
- return type == CacheObject.TOMBSTONE;
- }
- }
+ if (dataIo instanceof AbstractDataPageIO) {
+ DataPagePayload payload = ((AbstractDataPageIO<?>)dataIo).readPayload(dataBufAddr, itemId(link), pageSize);
- return false;
- });
- }
- catch (Exception e) {
- nodeCtx.errors.computeIfAbsent(pageId, k -> new LinkedList<>()).add(e);
+ if (payload.offset() <= 0 || payload.payloadSize() <= 0) {
+ throw new IgniteException(new GridStringBuilder("Invalid data page payload: ")
+ .a("off=").a(payload.offset())
+ .a(", size=").a(payload.payloadSize())
+ .a(", nextLink=").a(payload.nextLink())
+ .toString());
+ }
}
- }
- return new CacheAwareLink(cacheId, link, tombstone);
- }
- else
- throw new IgniteException("Unexpected page io: " + io.getClass().getSimpleName());
- }
+ return null;
+ });
+ }, ctx, PageIO.getPageId(addr));
- /** */
- private boolean isLinkIo(PageIO io) {
- return io instanceof InlineIO || io instanceof PendingRowIO || io instanceof RowLinkIO;
+ return new CacheAwareLink(cacheId, link);
}
/** */
- private long getLink(BPlusLeafIO<?> io, long addr, int idx) {
+ private long link(BPlusLeafIO<?> io, long addr, int idx) {
if (io instanceof RowLinkIO)
return ((RowLinkIO)io).getLink(addr, idx);
- if (io instanceof InlineIO)
+ else if (io instanceof InlineIO)
return ((InlineIO)io).link(addr, idx);
else if (io instanceof PendingRowIO)
return ((PendingRowIO)io).getLink(addr, idx);
else
throw new IgniteException("No link to data page on idx=" + idx);
}
-
- /** {@inheritDoc} */
- @Override public TreeNode getNode(PageContent content, long pageId, TreeTraverseContext nodeCtx) {
- if (nodeCtx.leafCb != null)
- nodeCtx.leafCb.cb(content, pageId);
-
- if (nodeCtx.itemCb != null) {
- for (Object item : content.items)
- nodeCtx.itemCb.cb(pageId, item, 0);
- }
-
- return new TreeNode(pageId, content.io, content.info, Collections.emptyList());
- }
}
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactory.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactory.java
index 769684c907f..c4b1518b8ac 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactory.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactory.java
@@ -18,12 +18,12 @@
package org.apache.ignite.internal.commandline.indexreader;
import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.Collection;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
import org.jetbrains.annotations.Nullable;
import static java.lang.String.format;
@@ -36,18 +36,43 @@ import static org.apache.ignite.internal.processors.cache.persistence.file.FileP
/**
* Factory {@link FilePageStore} for analyzing partition and index files.
*/
-public interface IgniteIndexReaderFilePageStoreFactory {
+class IgniteIndexReaderFilePageStoreFactory {
+ /** Directory with data(partitions and index). */
+ private final File dir;
+
+ /** {@link FilePageStore} factory by page store version. */
+ private final FileVersionCheckingFactory storeFactory;
+
+ /** Page size. */
+ private final int pageSize;
+
+ /** Partition count. */
+ private final int partCnt;
+
/**
- * Creating new {@link FilePageStore}. It can return {@code null} if partition file were not found,
- * for example: node should not contain it by affinity.
+ * Constructor.
*
- * @param partId Partition ID.
- * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}.
- * @param errors Errors while reading partition.
- * @return New instance of {@link FilePageStore} or {@code null}.
- * @throws IgniteCheckedException If there are errors when creating {@link FilePageStore}.
+ * @param dir Directory with data(partitions and index).
+ * @param pageSize Page size.
+ * @param partCnt Partition count.
+ * @param filePageStoreVer Page store version.
*/
- @Nullable FilePageStore createFilePageStore(int partId, byte type, Collection<Throwable> errors) throws IgniteCheckedException;
+ IgniteIndexReaderFilePageStoreFactory(File dir, int pageSize, int partCnt, int filePageStoreVer) {
+ this.dir = dir;
+ this.pageSize = pageSize;
+ this.partCnt = partCnt;
+
+ storeFactory = new FileVersionCheckingFactory(
+ new AsyncFileIOFactory(),
+ new AsyncFileIOFactory(),
+ () -> pageSize
+ ) {
+ /** {@inheritDoc} */
+ @Override public int latestVersion() {
+ return filePageStoreVer;
+ }
+ };
+ }
/**
* Creating new {@link FilePageStore} and initializing it.
@@ -55,31 +80,20 @@ public interface IgniteIndexReaderFilePageStoreFactory {
*
* @param partId Partition ID.
* @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}.
- * @param errors Errors while reading partition.
* @return New instance of {@link FilePageStore} or {@code null}.
* @throws IgniteCheckedException If there are errors when creating or initializing {@link FilePageStore}.
*/
- @Nullable default FilePageStore createFilePageStoreWithEnsure(
- int partId,
- byte type,
- Collection<Throwable> errors
- ) throws IgniteCheckedException {
- FilePageStore filePageStore = createFilePageStore(partId, type, errors);
+ @Nullable FilePageStore createFilePageStore(int partId, byte type) throws IgniteCheckedException {
+ File file = getFile(dir, partId, null);
- if (nonNull(filePageStore))
- filePageStore.ensure();
+ if (!file.exists())
+ return null;
- return filePageStore;
- }
+ FilePageStore filePageStore = (FilePageStore)storeFactory.createPageStore(type, file, l -> {});
- /**
- * Create buffer with header.
- *
- * @param type Data type, can be {@link PageIdAllocator#FLAG_IDX} or {@link PageIdAllocator#FLAG_DATA}.
- * @return New buffer with header.
- */
- default ByteBuffer headerBuffer(byte type) throws IgniteCheckedException {
- throw new UnsupportedOperationException();
+ filePageStore.ensure();
+
+ return filePageStore;
}
/**
@@ -90,7 +104,7 @@ public interface IgniteIndexReaderFilePageStoreFactory {
* @param fileExt File extension if it differs from {@link FilePageStoreManager#FILE_SUFFIX}.
* @return Partition or index file that may not exist.
*/
- default File getFile(File dir, int partId, @Nullable String fileExt) {
+ private File getFile(File dir, int partId, @Nullable String fileExt) {
String fileName = partId == INDEX_PARTITION ? INDEX_FILE_NAME : format(PART_FILE_TEMPLATE, partId);
if (nonNull(fileExt) && !FILE_SUFFIX.equals(fileExt))
@@ -104,12 +118,16 @@ public interface IgniteIndexReaderFilePageStoreFactory {
*
* @return Page size.
*/
- int pageSize();
+ int pageSize() {
+ return pageSize;
+ }
/**
* Return partition count.
*
* @return Partition count.
*/
- int partitionCount();
+ int partitionCount() {
+ return partCnt;
+ }
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactoryImpl.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactoryImpl.java
deleted file mode 100644
index b042792395a..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderFilePageStoreFactoryImpl.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.nio.file.Path;
-import java.util.Collection;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
-import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
-import org.apache.ignite.internal.processors.cache.persistence.file.FileVersionCheckingFactory;
-import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric;
-import org.apache.ignite.lang.IgniteOutClosure;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Factory {@link FilePageStore} for case of regular pds.
- */
-public class IgniteIndexReaderFilePageStoreFactoryImpl implements IgniteIndexReaderFilePageStoreFactory {
- /** Directory with data(partitions and index). */
- private final File dir;
-
- /** {@link FilePageStore} factory by page store version. */
- private final FileVersionCheckingFactory storeFactory;
-
- /** Metrics updater. */
- private final LongAdderMetric allocationTracker = new LongAdderMetric("n", "d");
-
- /** Page size. */
- private final int pageSize;
-
- /** Partition count. */
- private final int partCnt;
-
- /**
- * Constructor.
- *
- * @param dir Directory with data(partitions and index).
- * @param pageSize Page size.
- * @param partCnt Partition count.
- * @param filePageStoreVer Page store version.
- */
- public IgniteIndexReaderFilePageStoreFactoryImpl(File dir, int pageSize, int partCnt, int filePageStoreVer) {
- this.dir = dir;
- this.pageSize = pageSize;
- this.partCnt = partCnt;
-
- storeFactory = new FileVersionCheckingFactory(
- new AsyncFileIOFactory(),
- new AsyncFileIOFactory(),
- () -> pageSize
- ) {
- /** {@inheritDoc} */
- @Override public int latestVersion() {
- return filePageStoreVer;
- }
- };
- }
-
- /** {@inheritDoc} */
- @Override @Nullable public FilePageStore createFilePageStore(int partId, byte type, Collection<Throwable> errors)
- throws IgniteCheckedException {
- File file = getFile(dir, partId, null);
-
- return !file.exists() ? null : (FilePageStore)storeFactory.createPageStore(type, file, allocationTracker::add);
- }
-
- /** {@inheritDoc} */
- @Override public ByteBuffer headerBuffer(byte type) throws IgniteCheckedException {
- int ver = storeFactory.latestVersion();
-
- FilePageStore store =
- (FilePageStore)storeFactory.createPageStore(type, (IgniteOutClosure<Path>)null, allocationTracker::add);
-
- return store.header(type, storeFactory.headerSize(ver));
- }
-
- /** {@inheritDoc} */
- @Override public int pageSize() {
- return pageSize;
- }
-
- /** {@inheritDoc} */
- @Override public int partitionCount() {
- return partCnt;
- }
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IndexReaderUtils.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IndexReaderUtils.java
deleted file mode 100644
index 74b4e005bf5..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/IndexReaderUtils.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-import java.io.File;
-
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static java.lang.String.format;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.INDEX_FILE_NAME;
-import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.PART_FILE_TEMPLATE;
-
-/**
- * Collection of utility methods used throughout index reading.
- */
-public class IndexReaderUtils {
- /**
- * Private constructor.
- */
- private IndexReaderUtils() {
- throw new UnsupportedOperationException();
- }
-
- /**
- * Align value to base.
- *
- * @param val Value.
- * @param base Base.
- * @return Base-aligned value.
- */
- public static long align(long val, int base) {
- if ((val % base) == 0)
- return val;
- else
- return ((val / base) + 1) * base;
- }
-
- /**
- * Returns approximate link size in bytes.
- *
- * @return Approximate link size in bytes.
- */
- public static long linkSize() {
- return U.jvm32Bit() ? 4 : 8;
- }
-
- /**
- * Approximate calculation of object size in bytes.
- *
- * @param fieldsSize Approximate fields size in bytes.
- * @return Approximate object size in bytes.
- */
- public static long objectSize(long fieldsSize) {
- return align(/*Object header*/(2 * linkSize()) + fieldsSize, 8);
- }
-
- /**
- * Approximate calculation of array objects size in bytes.
- *
- * @param len Length of array.
- * @return Approximate array objects size in bytes.
- */
- public static long objectArraySize(int len) {
- return align(/*Object header*/(2 * linkSize()) + (linkSize() * len), 8);
- }
-
- /**
- * Returns the absolute pathname string of partition.
- *
- * @param partId Partition number.
- * @param parent Partition directory.
- * @return Partition absolute pathname string.
- */
- public static String partitionFileAbsolutePath(File parent, int partId) {
- String fileName = partId == INDEX_PARTITION ? INDEX_FILE_NAME : format(PART_FILE_TEMPLATE, partId);
-
- return new File(parent, fileName).getAbsolutePath();
- }
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/ItemCallback.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/ItemCallback.java
deleted file mode 100644
index 6e20a996adb..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/ItemCallback.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-/**
- * Callback that will be executed for each tree item while traversing, if it's set.
- */
-interface ItemCallback {
- /** */
- void cb(long currPageId, Object item, long link);
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageCallback.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageCallback.java
deleted file mode 100644
index 8a6a7d0b946..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageCallback.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-/**
- * Callback that will be executed for each page of the tree while traversing, if it's set.
- */
-interface PageCallback {
- /** */
- void cb(PageContent pageContent, long pageId);
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageContent.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageContent.java
deleted file mode 100644
index f3aecc56581..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageContent.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-import java.util.List;
-
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-
-/**
- * Content of the deserialized page. When content is gained, we can free the page buffer.
- */
-class PageContent {
- /** */
- final PageIO io;
-
- /** List of children page ids, or links to root pages (for meta leaf). */
- final List<Long> linkedPageIds;
-
- /** List of items (for leaf pages). */
- final List<Object> items;
-
- /** Some info. */
- final String info;
-
- /** */
- public PageContent(PageIO io, List<Long> linkedPageIds, List<Object> items, String info) {
- this.io = io;
- this.linkedPageIds = linkedPageIds;
- this.items = items;
- this.info = info;
- }
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java
index f3b5df48421..84d5cfc0f1f 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/PageListsInfo.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.commandline.indexreader;
import java.util.List;
import java.util.Map;
-import java.util.Set;
import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
@@ -35,25 +34,25 @@ class PageListsInfo {
*/
final Map<IgniteBiTuple<Long, Integer>, List<Long>> bucketsData;
- /** All page ids from page lists. */
- final Set<Long> allPages;
+ /** Found pages count. */
+ final long pagesCnt;
/** Page type statistics. */
- final Map<Class<? extends PageIO>, Long> pageListStat;
+ final Map<Class<? extends PageIO>, Long> ioStat;
/** Map of errors, pageId -> list of exceptions. */
- final Map<Long, List<Throwable>> errors;
+ final Map<Long, List<String>> errors;
/** */
public PageListsInfo(
- Map<IgniteBiTuple<Long, Integer>, List<Long>> bucketsData,
- Set<Long> allPages,
- Map<Class<? extends PageIO>, Long> pageListStat,
- Map<Long, List<Throwable>> errors
+ Map<IgniteBiTuple<Long, Integer>, List<Long>> bucketsData,
+ long pagesCnt,
+ Map<Class<? extends PageIO>, Long> ioStat,
+ Map<Long, List<String>> errors
) {
this.bucketsData = bucketsData;
- this.allPages = allPages;
- this.pageListStat = pageListStat;
+ this.pagesCnt = pagesCnt;
+ this.ioStat = ioStat;
this.errors = errors;
}
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/ScanContext.java
similarity index 57%
rename from modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java
rename to modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/ScanContext.java
index b30aaaee186..c06f43a03bd 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraverseContext.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/ScanContext.java
@@ -17,24 +17,18 @@
package org.apache.ignite.internal.commandline.indexreader;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-import org.jetbrains.annotations.Nullable;
-
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.getCacheId;
/**
* Traverse context, which is used for tree traversal and is unique for traversal of one single tree.
*/
-class TreeTraverseContext {
- /** Tree name. */
- final String treeName;
-
- /** Cache id. */
+class ScanContext {
+ /** Cache id or {@code -1} for sequential scan. */
final int cacheId;
/** Page store. */
@@ -44,32 +38,32 @@ class TreeTraverseContext {
final Map<Class<? extends PageIO>, Long> ioStat;
/** Map of errors, pageId -> set of exceptions. */
- final Map<Long, List<Throwable>> errors;
+ final Map<Long, List<String>> errors;
- /** Callback that is called for each inner node page. */
- @Nullable final PageCallback innerCb;
+ /** List of items storage. */
+ final ItemStorage items;
- /** Callback that is called for each leaf node page.*/
- @Nullable final PageCallback leafCb;
+ /** */
+ public ScanContext(int cacheId, FilePageStore store, ItemStorage items) {
+ this.cacheId = cacheId;
+ this.store = store;
+ this.items = items;
+ this.ioStat = new LinkedHashMap<>();
+ this.errors = new LinkedHashMap<>();
+ }
- /** Callback that is called for each leaf item. */
- @Nullable final ItemCallback itemCb;
+ /** */
+ public void onPageIO(PageIO io) {
+ onPageIO(io.getClass(), ioStat, 1);
+ }
/** */
- public TreeTraverseContext(
- String treeName,
- FilePageStore store,
- @Nullable PageCallback innerCb,
- @Nullable PageCallback leafCb,
- @Nullable ItemCallback itemCb
- ) {
- this.treeName = treeName;
- this.cacheId = getCacheId(treeName);
- this.store = store;
- this.ioStat = new HashMap<>();
- this.errors = new HashMap<>();
- this.innerCb = innerCb;
- this.leafCb = leafCb;
- this.itemCb = itemCb;
+ public static void onPageIO(Class<? extends PageIO> io, Map<Class<? extends PageIO>, Long> ioStat, long cnt) {
+ ioStat.compute(io, (k, v) -> v == null ? cnt : v + cnt);
+ }
+
+ /** */
+ public void onLeafPage(long pageId, List<Object> data) {
+ data.forEach(items::add);
}
}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeNode.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeNode.java
deleted file mode 100644
index e5e844b5939..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeNode.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-import java.util.List;
-
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-
-/**
- * Tree node info. It is used to represent tree nodes in recursive traversal.
- */
-class TreeNode {
- /** */
- final long pageId;
-
- /** */
- final PageIO io;
-
- /** */
- final String additionalInfo;
-
- /** */
- final List<TreeNode> children;
-
- /** */
- public TreeNode(long pageId, PageIO io, String additionalInfo, List<TreeNode> children) {
- this.pageId = pageId;
- this.io = io;
- this.additionalInfo = additionalInfo;
- this.children = children;
- }
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java
deleted file mode 100644
index 6e49c565156..00000000000
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/indexreader/TreeTraversalInfo.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.commandline.indexreader;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
-
-/**
- * This class is used to store result of tree traversal.
- */
-public class TreeTraversalInfo {
- /** Page type statistics. */
- final Map<Class<? extends PageIO>, Long> ioStat;
-
- /** Map of errors, pageId -> set of exceptions. */
- final Map<Long, List<Throwable>> errors;
-
- /** Set of all inner page ids. */
- final Set<Long> innerPageIds;
-
- /** Root page id. */
- final long rootPageId;
-
- /**
- * List of items storage.
- */
- final ItemStorage itemStorage;
-
- /** */
- public TreeTraversalInfo(
- Map<Class<? extends PageIO>, Long> ioStat,
- Map<Long, List<Throwable>> errors,
- Set<Long> innerPageIds,
- long rootPageId,
- ItemStorage itemStorage
- ) {
- this.ioStat = ioStat;
- this.errors = errors;
- this.innerPageIds = innerPageIds;
- this.rootPageId = rootPageId;
- this.itemStorage = itemStorage;
- }
-}
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/systemview/SystemViewCommand.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/systemview/SystemViewCommand.java
index 9a3f72d5183..f9d53caf5d6 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/systemview/SystemViewCommand.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/systemview/SystemViewCommand.java
@@ -102,7 +102,12 @@ public class SystemViewCommand extends AbstractCommand<VisorSystemViewTaskArg> {
* @param log Logger.
*/
public static void printTable(List<String> titles, List<SimpleType> types, List<List<?>> data, Logger log) {
- List<Integer> colSzs = titles.stream().map(String::length).collect(Collectors.toList());
+ List<Integer> colSzs;
+
+ if (titles != null)
+ colSzs = titles.stream().map(String::length).collect(Collectors.toList());
+ else
+ colSzs = types.stream().map(x -> 0).collect(Collectors.toList());
List<List<String>> rows = new ArrayList<>(data.size());
@@ -118,7 +123,8 @@ public class SystemViewCommand extends AbstractCommand<VisorSystemViewTaskArg> {
}).collect(Collectors.toList()));
});
- printRow(titles, nCopies(titles.size(), STRING), colSzs, log);
+ if (titles != null)
+ printRow(titles, nCopies(titles.size(), STRING), colSzs, log);
rows.forEach(row -> printRow(row, types, colSzs, log));
}
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java
index f7e51ae2bf6..0b6358337f3 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/internal/commandline/indexreader/IgniteIndexReaderTest.java
@@ -17,16 +17,14 @@
package org.apache.ignite.internal.commandline.indexreader;
-import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
-import java.io.OutputStream;
-import java.io.PrintStream;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
@@ -34,6 +32,8 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Handler;
+import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
@@ -49,7 +49,10 @@ import org.apache.ignite.configuration.DataStorageConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
+import org.apache.ignite.internal.commandline.ProgressPrinter;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.processors.cache.persistence.IndexStorageImpl;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.query.QueryUtils;
@@ -59,7 +62,7 @@ import org.apache.ignite.internal.util.lang.IgnitePair;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.util.GridCommandHandlerAbstractTest;
import org.jetbrains.annotations.Nullable;
import org.junit.Test;
@@ -75,10 +78,9 @@ import static java.util.regex.Pattern.compile;
import static java.util.stream.Collectors.joining;
import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.ERROR_PREFIX;
import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.HORIZONTAL_SCAN_NAME;
+import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.META_TREE_NAME;
import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.RECURSIVE_TRAVERSE_NAME;
import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.normalizePageId;
-import static org.apache.ignite.internal.commandline.indexreader.IgniteIndexReader.partMetaPageId;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
import static org.apache.ignite.internal.pagemem.PageIdUtils.pageIndex;
import static org.apache.ignite.internal.pagemem.PageIdUtils.partId;
@@ -92,7 +94,7 @@ import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
/**
* Class for testing {@link IgniteIndexReader}.
*/
-public class IgniteIndexReaderTest extends GridCommonAbstractTest {
+public class IgniteIndexReaderTest extends GridCommandHandlerAbstractTest {
/** Page size. */
protected static final int PAGE_SIZE = 4096;
@@ -126,9 +128,9 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
/** Common part of regexp for single index output validation. */
private static final String CHECK_IDX_PTRN_COMMON =
"<PREFIX>Index tree: I \\[idxName=[\\-_0-9]{1,20}_%s##H2Tree.0, pageId=[0-9a-f]{16}\\]" +
- LINE_DELIM + "<PREFIX>-- Page stat:" +
- LINE_DELIM + "<PREFIX>([0-9a-zA-Z]{1,50}: [0-9]{1,5}" +
- LINE_DELIM + "<PREFIX>){%s,1000}-- Count of items found in leaf pages: %s" +
+ LINE_DELIM + "<PREFIX>---- Page stat:" +
+ LINE_DELIM + "<PREFIX>([0-9a-zA-Z]{1,50}.*[0-9]{1,5}" +
+ LINE_DELIM + "<PREFIX>){%s,1000}---- Count of items found in leaf pages: %s" +
LINE_DELIM;
/** Regexp to validate output of correct index. */
@@ -139,7 +141,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
private static final String CHECK_IDX_PTRN_WITH_ERRORS =
CHECK_IDX_PTRN_COMMON + "<PREFIX>" + ERROR_PREFIX + "Errors:" +
LINE_DELIM + "<PREFIX>" + ERROR_PREFIX + "Page id=[0-9]{1,30}, exceptions:" +
- LINE_DELIM + "class.*?Exception.*";
+ LINE_DELIM + "Failed to read page.*";
/** Work directory, containing cache group directories. */
private static File workDir;
@@ -153,6 +155,13 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
prepareWorkDir();
}
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ injectTestSystemOut();
+ }
+
/** {@inheritDoc} */
@Override protected void afterTestsStopped() throws Exception {
super.afterTestsStopped();
@@ -268,27 +277,36 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
private IgniteBiTuple<Long, Long> findPagesForAnyCacheKey(File workDir, String cacheGrp) throws IgniteCheckedException {
File dir = new File(workDir, dataDir(cacheGrp));
- try (IgniteIndexReader reader = new IgniteIndexReader(null, false, null, createFilePageStoreFactory(dir))) {
- IgniteBiTuple<Long, Long> partitionRoots = reader.partitionRoots(partMetaPageId(INDEX_PARTITION, FLAG_IDX));
+ // Take any inner page from tree.
+ AtomicLong anyLeafId = new AtomicLong();
+
+ IgniteIndexReader reader0 = new IgniteIndexReader(null, false, createFilePageStoreFactory(dir), createTestLogger()) {
+ @Override ScanContext createContext(int cacheId, FilePageStore store, ItemStorage items) {
+ return new ScanContext(cacheId, store, items) {
+ @Override public void onLeafPage(long pageId, List<Object> data) {
+ super.onLeafPage(pageId, data);
+
+ anyLeafId.set(pageId);
+ }
+ };
+ }
+ };
+
+ try (IgniteIndexReader reader = reader0) {
+ long[] partitionRoots = reader.partitionRoots(PageIdAllocator.META_PAGE_ID);
ItemsListStorage<IndexStorageImpl.IndexItem> idxItemStorage = new ItemsListStorage<>();
- reader.traverseTree(partitionRoots.get1(), "MetaTree", idxItemStorage);
+ reader.recursiveTreeScan(partitionRoots[0], META_TREE_NAME, idxItemStorage);
// Take any index item.
IndexStorageImpl.IndexItem idxItem = idxItemStorage.iterator().next();
ItemsListStorage<CacheAwareLink> linkStorage = new ItemsListStorage<>();
- // Take any inner page from tree.
- AtomicLong anyLeafId = new AtomicLong();
-
- reader.traverseTree(
+ reader.recursiveTreeScan(
normalizePageId(idxItem.pageId()),
idxItem.nameString(),
- null,
- (c, pageId) -> anyLeafId.set(pageId),
- null,
linkStorage
);
@@ -583,11 +601,11 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
boolean partReadingErr,
boolean idxSizeConsistent
) {
- assertContains(log, output, RECURSIVE_TRAVERSE_NAME + "Total trees: " + treesCnt);
- assertContains(log, output, HORIZONTAL_SCAN_NAME + "Total trees: " + treesCnt);
- assertContains(log, output, RECURSIVE_TRAVERSE_NAME + "Total errors during trees traversal: " + travErrCnt);
- assertContains(log, output, HORIZONTAL_SCAN_NAME + "Total errors during trees traversal: " + travErrCnt);
- assertContains(log, output, "Total errors during lists scan: " + pageListsErrCnt);
+ assertFound(output, RECURSIVE_TRAVERSE_NAME + "Total trees:.*" + treesCnt);
+ assertFound(output, HORIZONTAL_SCAN_NAME + "Total trees:.*" + treesCnt);
+ assertFound(output, RECURSIVE_TRAVERSE_NAME + "Total errors during trees traversal:.*" + travErrCnt);
+ assertFound(output, HORIZONTAL_SCAN_NAME + "Total errors during trees traversal:.*" + travErrCnt);
+ assertFound(output, "Total errors during lists scan:.*" + pageListsErrCnt);
if (idxSizeConsistent)
assertContains(log, output, "No index size consistency errors found.");
@@ -595,7 +613,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
assertContains(log, output, "Index size inconsistency");
if (seqErrCnt >= 0)
- assertContains(log, output, "Total errors occurred during sequential scan: " + seqErrCnt);
+ assertFound(output, "Total errors occurred during sequential scan:.*" + seqErrCnt);
else
assertContains(log, output, "Orphan pages were not reported due to --indexes filter.");
@@ -695,22 +713,29 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
@Nullable String[] idxs,
boolean checkParts
) throws IgniteCheckedException {
- File dir = new File(workDir, dataDir(cacheGrp));
-
- OutputStream destStream = new ByteArrayOutputStream();
+ testOut.reset();
- List<String> idxList = isNull(idxs) ? null : asList(idxs);
+ Logger logger = createTestLogger();
- try (IgniteIndexReader reader = new IgniteIndexReader(
- isNull(idxList) ? null : idx -> idxList.stream().anyMatch(idx::endsWith),
+ IgniteIndexReader reader0 = new IgniteIndexReader(
+ isNull(idxs) ? null : idx -> Arrays.stream(idxs).anyMatch(idx::endsWith),
checkParts,
- new PrintStream(destStream),
- createFilePageStoreFactory(dir)
- )) {
- reader.readIdx();
+ createFilePageStoreFactory(new File(workDir, dataDir(cacheGrp))),
+ logger
+ ) {
+ /** {@inheritDoc} */
+ @Override ProgressPrinter createProgressPrinter(String caption, long total) {
+ return new ProgressPrinter(System.err, caption, total);
+ }
+ };
+ try (IgniteIndexReader reader = reader0) {
+ reader.readIndex();
}
- return destStream.toString();
+ // Flush all Logger handlers to make log data available to test.
+ Arrays.stream(logger.getHandlers()).forEach(Handler::flush);
+
+ return testOut.toString();
}
/**
@@ -721,7 +746,7 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
protected IgniteIndexReaderFilePageStoreFactory createFilePageStoreFactory(
File dir
) {
- return new IgniteIndexReaderFilePageStoreFactoryImpl(dir, PAGE_SIZE, PART_CNT, PAGE_STORE_VER);
+ return new IgniteIndexReaderFilePageStoreFactory(dir, PAGE_SIZE, PART_CNT, PAGE_STORE_VER);
}
/**
@@ -956,13 +981,8 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
String output = runIndexReader(workDirs.get(0), CACHE_GROUP_NAME, null, true);
// Pattern with errors count > 9
- Pattern ptrn = compile(
- "Partition check finished, total errors: [0-9]{2,5}, total problem partitions: [0-9]{2,5}"
- );
-
- assertTrue(output, ptrn.matcher(output).find());
-
- assertContains(log, output, "Total errors during lists scan: 0");
+ assertFound(output, "Partition check finished, total errors: [0-9]{2,5}, total problem partitions: [0-9]{2,5}");
+ assertFound(output, "Total errors during lists scan:.*0");
}
finally {
for (File dir : workDirs)
@@ -970,6 +990,11 @@ public class IgniteIndexReaderTest extends GridCommonAbstractTest {
}
}
+ /** */
+ private void assertFound(String output, String pattern) {
+ assertTrue(output, compile(pattern).matcher(output).find());
+ }
+
/**
* Checks whether corrupted pages are found in index.
* Index will be corrupted for work directories, and first work directory will be used to start {@link IgniteIndexReader}.
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
index 077a553dc50..67552e27b0b 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerAbstractTest.java
@@ -200,7 +200,7 @@ public abstract class GridCommandHandlerAbstractTest extends GridCommonAbstractT
/**
* @return Logger.
*/
- protected Logger createTestLogger() {
+ public static Logger createTestLogger() {
Logger log = CommandHandler.initLogger(null);
// Adding logging to console.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
index 9c89916c476..38636716c22 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgnitionEx.java
@@ -150,6 +150,7 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
import static org.apache.ignite.internal.IgniteComponentType.SPRING;
+import static org.apache.ignite.internal.util.IgniteUtils.EMPTY_STRS;
import static org.apache.ignite.plugin.segmentation.SegmentationPolicy.RESTART_JVM;
/**
@@ -1591,9 +1592,6 @@ public class IgnitionEx {
private static final Map<MBeanServer, GridMBeanServerData> mbeans =
new HashMap<>();
- /** */
- private static final String[] EMPTY_STR_ARR = new String[0];
-
/** Grid name. */
private final String name;
@@ -2014,7 +2012,7 @@ public class IgnitionEx {
myCfg.setMarshaller(marsh);
if (myCfg.getPeerClassLoadingLocalClassPathExclude() == null)
- myCfg.setPeerClassLoadingLocalClassPathExclude(EMPTY_STR_ARR);
+ myCfg.setPeerClassLoadingLocalClassPathExclude(EMPTY_STRS);
initializeDefaultSpi(myCfg);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java
index 41c9d8dcc4a..0f21c77c438 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheObject.java
@@ -38,9 +38,6 @@ public interface CacheObject extends Message {
/** */
public static final byte TYPE_BINARY_ENUM = 101;
- /** */
- public static final byte TOMBSTONE = -1;
-
/**
* @param ctx Context.
* @param cpy If {@code true} need to copy value.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
index 13a6dec9494..552ff4702e3 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteUtils.java
@@ -366,6 +366,9 @@ public abstract class IgniteUtils {
/** Empty longs. */
public static final long[] EMPTY_LONGS = new long[0];
+ /** */
+ public static final String[] EMPTY_STRS = new String[0];
+
/** Empty longs. */
public static final Field[] EMPTY_FIELDS = new Field[0];
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/RunnableX.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/RunnableX.java
new file mode 100644
index 00000000000..af8453a67e9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/RunnableX.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.util.lang;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.lang.IgniteRunnable;
+
+/**
+ * Runnable that can throw checked exception.
+ * Any exception from {@link #runx()} will be converted to {@link IgniteException}.
+ */
+@FunctionalInterface
+public interface RunnableX extends IgniteRunnable {
+ /**
+ * @throws Exception If failed.
+ */
+ void runx() throws Exception;
+
+ /** {@inheritDoc} */
+ @Override default void run() {
+ try {
+ runx();
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/metric/JmxExporterSpiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/metric/JmxExporterSpiTest.java
index e14287cfa91..999b3651074 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/metric/JmxExporterSpiTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/metric/JmxExporterSpiTest.java
@@ -83,7 +83,6 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.services.ServiceConfiguration;
import org.apache.ignite.spi.metric.jmx.JmxMetricExporterSpi;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.apache.ignite.transactions.Transaction;
import org.junit.Test;
@@ -250,8 +249,8 @@ public class JmxExporterSpiTest extends AbstractExporterSpiTest {
public void testFilterAndExport() throws Exception {
createAdditionalMetrics(ignite);
- assertThrowsWithCause(new RunnableX() {
- @Override public void runx() throws Exception {
+ assertThrowsWithCause(new Runnable() {
+ @Override public void run() {
metricRegistry(ignite.name(), "filtered", "metric");
}
}, IgniteException.class);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheCreateDestroyClusterReadOnlyModeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheCreateDestroyClusterReadOnlyModeTest.java
index 7587bf8c67b..919a5641541 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheCreateDestroyClusterReadOnlyModeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheCreateDestroyClusterReadOnlyModeTest.java
@@ -23,8 +23,8 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.cluster.ClusterState;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
import static java.util.Arrays.asList;
@@ -178,7 +178,7 @@ public class CacheCreateDestroyClusterReadOnlyModeTest extends CacheCreateDestro
}
/** */
- private void executeAndCheckException(GridTestUtils.RunnableX clo, String cacheName) {
+ private void executeAndCheckException(RunnableX clo, String cacheName) {
Throwable ex = assertThrows(log, clo, Exception.class, null);
ClusterReadOnlyModeTestUtils.checkRootCause(ex, cacheName);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheProcessorActiveTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheProcessorActiveTxTest.java
index 3099f742f6a..65764e37970 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheProcessorActiveTxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheProcessorActiveTxTest.java
@@ -21,7 +21,7 @@ import java.util.List;
import org.apache.ignite.IgniteException;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.junit.Test;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBlockOnReadAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBlockOnReadAbstractTest.java
index bbfc17311ce..23fc90ceb6c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBlockOnReadAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBlockOnReadAbstractTest.java
@@ -66,6 +66,7 @@ import org.apache.ignite.internal.processors.cache.ExchangeActions.CacheActionDa
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage;
import org.apache.ignite.internal.processors.cluster.ChangeGlobalStateMessage;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiPredicate;
@@ -77,7 +78,6 @@ import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryAbstractMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeAddFinishedMessage;
import org.apache.ignite.spi.discovery.tcp.messages.TcpDiscoveryNodeLeftMessage;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.jetbrains.annotations.NotNull;
import org.junit.Test;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/ComputePermissionCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/ComputePermissionCheckTest.java
index c2a7b1eb71e..042c002511d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/ComputePermissionCheckTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/ComputePermissionCheckTest.java
@@ -36,6 +36,7 @@ import org.apache.ignite.compute.ComputeJobResult;
import org.apache.ignite.compute.ComputeJobResultPolicy;
import org.apache.ignite.compute.ComputeTask;
import org.apache.ignite.internal.processors.security.AbstractSecurityTest;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteRunnable;
@@ -43,7 +44,6 @@ import org.apache.ignite.plugin.security.SecurityException;
import org.apache.ignite.plugin.security.SecurityPermission;
import org.apache.ignite.plugin.security.SecurityPermissionSet;
import org.apache.ignite.plugin.security.SecurityPermissionSetBuilder;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.jetbrains.annotations.Nullable;
import org.junit.Test;
import org.junit.runner.RunWith;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/closure/ExecutorServiceRemoteSecurityContextCheckTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/closure/ExecutorServiceRemoteSecurityContextCheckTest.java
index 7d230d3dc72..dc2b92f04ea 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/closure/ExecutorServiceRemoteSecurityContextCheckTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/compute/closure/ExecutorServiceRemoteSecurityContextCheckTest.java
@@ -23,8 +23,8 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.Ignition;
import org.apache.ignite.cluster.ClusterState;
import org.apache.ignite.internal.processors.security.AbstractRemoteSecurityContextCheckTest;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.testframework.GridTestUtils.IgniteRunnableX;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@@ -62,7 +62,7 @@ public class ExecutorServiceRemoteSecurityContextCheckTest extends AbstractRemot
/** */
@Test
public void test() {
- runAndCheck((IgniteRunnableX)() -> {
+ runAndCheck((RunnableX)() -> {
Ignite loc = Ignition.localIgnite();
for (UUID nodeId : nodesToCheckIds()) {
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractContinuousQuerySandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractContinuousQuerySandboxTest.java
index cd5b0377bc0..6aed72820dd 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractContinuousQuerySandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractContinuousQuerySandboxTest.java
@@ -29,8 +29,8 @@ import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.query.Query;
import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.testframework.GridTestUtils;
/**
* Abstract class to test that a remote filter and transformer of ContinuousQueries run on a remote node inside the
@@ -81,7 +81,7 @@ public class AbstractContinuousQuerySandboxTest extends AbstractSandboxTest {
}
/** */
- private GridTestUtils.RunnableX operation(String nodeName,
+ private RunnableX operation(String nodeName,
Supplier<Query<Cache.Entry<Integer, Integer>>> s, boolean init) {
return () -> {
error = null;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractSandboxTest.java
index 32d92a567a5..943124f07c2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/AbstractSandboxTest.java
@@ -33,8 +33,8 @@ import javax.tools.JavaCompiler;
import javax.tools.ToolProvider;
import org.apache.ignite.Ignite;
import org.apache.ignite.internal.processors.security.AbstractSecurityTest;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.testframework.GridTestUtils;
import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.ALLOW_ALL;
import static org.apache.ignite.testframework.GridTestUtils.assertThrowsWithCause;
@@ -127,7 +127,7 @@ public abstract class AbstractSandboxTest extends AbstractSecurityTest {
/**
* @param r RunnableX that that runs {@link AbstractSandboxTest#controlAction()}.
*/
- protected void runForbiddenOperation(GridTestUtils.RunnableX r, Class<? extends Throwable> cls) {
+ protected void runForbiddenOperation(RunnableX r, Class<? extends Throwable> cls) {
System.clearProperty(PROP_NAME);
assertThrowsWithCause(r, cls);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/CacheSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/CacheSandboxTest.java
index 1663ecf9344..865f63e6688 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/CacheSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/CacheSandboxTest.java
@@ -26,8 +26,8 @@ import org.apache.ignite.cache.CacheEntryProcessor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Test;
import static java.util.Collections.singleton;
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/ComputeSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/ComputeSandboxTest.java
index 5ae1b318544..7da48e665eb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/ComputeSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/ComputeSandboxTest.java
@@ -29,10 +29,10 @@ import org.apache.ignite.compute.ComputeJob;
import org.apache.ignite.compute.ComputeJobResult;
import org.apache.ignite.compute.ComputeJobResultPolicy;
import org.apache.ignite.compute.ComputeTask;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgniteCallable;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.testframework.GridTestUtils;
import org.jetbrains.annotations.Nullable;
import org.junit.Test;
@@ -81,7 +81,7 @@ public class ComputeSandboxTest extends AbstractSandboxTest {
/**
* @return Stream of Compute operations to test.
*/
- private Stream<GridTestUtils.RunnableX> computeOperations(Ignite node) {
+ private Stream<RunnableX> computeOperations(Ignite node) {
return Stream.of(
() -> node.compute().execute(COMPUTE_TASK, 0),
() -> node.compute().broadcast(CALLABLE),
@@ -100,7 +100,7 @@ public class ComputeSandboxTest extends AbstractSandboxTest {
/**
* @return Stream of ExecutorService operations to test.
*/
- private Stream<GridTestUtils.RunnableX> executorServiceOperations(Ignite node) {
+ private Stream<RunnableX> executorServiceOperations(Ignite node) {
return Stream.of(
() -> node.executorService().invokeAll(singletonList(CALLABLE))
.stream().findFirst().orElseThrow(IgniteException::new).get(),
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/DataStreamerSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/DataStreamerSandboxTest.java
index e842a167767..3a22b23b096 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/DataStreamerSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/DataStreamerSandboxTest.java
@@ -22,7 +22,7 @@ import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteDataStreamer;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.junit.Test;
/**
@@ -45,7 +45,7 @@ public class DataStreamerSandboxTest extends AbstractSandboxTest {
/**
* @return Operation to test.
*/
- private GridTestUtils.RunnableX operation(Ignite node) {
+ private RunnableX operation(Ignite node) {
return () -> {
try (IgniteDataStreamer<Integer, Integer> strm = node.dataStreamer(TEST_CACHE)) {
strm.receiver((cache, entries) -> controlAction());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/EventsSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/EventsSandboxTest.java
index 9d7682255fb..e842892670b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/EventsSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/EventsSandboxTest.java
@@ -32,10 +32,10 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.CacheEvent;
import org.apache.ignite.events.Event;
import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.resources.LoggerResource;
-import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
/**
@@ -97,7 +97,7 @@ public class EventsSandboxTest extends AbstractSandboxTest {
UUID id = func.apply(evts, cacheName);
try {
- GridTestUtils.RunnableX r = () -> {
+ RunnableX r = () -> {
error = null;
latch = new CountDownLatch(1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/MessagingSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/MessagingSandboxTest.java
index 4e48295be59..f12ada81170 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/MessagingSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/MessagingSandboxTest.java
@@ -24,8 +24,8 @@ import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteMessaging;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.lang.IgniteBiPredicate;
-import org.apache.ignite.testframework.GridTestUtils;
import org.junit.Test;
import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.ALLOW_ALL;
@@ -78,7 +78,7 @@ public class MessagingSandboxTest extends AbstractSandboxTest {
UUID listenerId = func.apply(messaging, topic);
try {
- GridTestUtils.RunnableX r = () -> {
+ RunnableX r = () -> {
error = null;
latch = new CountDownLatch(1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/SchedulerSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/SchedulerSandboxTest.java
index befcb88a995..a599c8b7f4e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/SchedulerSandboxTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/sandbox/SchedulerSandboxTest.java
@@ -24,7 +24,7 @@ import java.util.function.Consumer;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteScheduler;
import org.apache.ignite.Ignition;
-import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.junit.Test;
/**
@@ -85,7 +85,7 @@ public class SchedulerSandboxTest extends AbstractSandboxTest {
/** */
private void execute(Ignite node, Consumer<IgniteScheduler> consumer, boolean isForbiddenCase) {
- GridTestUtils.RunnableX r = () -> node.compute().run(() -> {
+ RunnableX r = () -> node.compute().run(() -> {
error = null;
latch = new CountDownLatch(1);
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/service/ServiceAuthorizationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/service/ServiceAuthorizationTest.java
index 5996d2a41bb..0e846afd0a4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/service/ServiceAuthorizationTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/service/ServiceAuthorizationTest.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.security.AbstractSecurityTest;
import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.plugin.security.SecurityException;
@@ -43,7 +44,6 @@ import org.apache.ignite.services.Service;
import org.apache.ignite.services.ServiceCallContext;
import org.apache.ignite.services.ServiceConfiguration;
import org.apache.ignite.services.ServiceDeploymentException;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
index 19c11f3796e..d9f8ade92ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/GridTestUtils.java
@@ -103,6 +103,7 @@ import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.lang.GridAbsClosure;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.internal.util.lang.IgnitePair;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.G;
import org.apache.ignite.internal.util.typedef.T2;
@@ -112,7 +113,6 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteRunnable;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.spi.discovery.DiscoveryNotification;
import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
@@ -2593,52 +2593,6 @@ public final class GridTestUtils {
}
}
- /**
- * Runnable that can throw exceptions.
- */
- @FunctionalInterface
- public interface RunnableX extends Runnable {
- /**
- * Runnable body.
- *
- * @throws Exception If failed.
- */
- void runx() throws Exception;
-
- /** {@inheritdoc} */
- @Override default void run() {
- try {
- runx();
- }
- catch (Exception e) {
- throw new IgniteException(e);
- }
- }
- }
-
- /**
- * IgniteRunnable that can throw exceptions.
- */
- @FunctionalInterface
- public interface IgniteRunnableX extends IgniteRunnable {
- /**
- * Runnable body.
- *
- * @throws Exception If failed.
- */
- void runx() throws Exception;
-
- /** {@inheritdoc} */
- @Override default void run() {
- try {
- runx();
- }
- catch (Exception e) {
- throw new IgniteException(e);
- }
- }
- }
-
/**
* @param runnableX Runnable with exception.
*/
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLoggerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLoggerSelfTest.java
index 0e64c7a11dd..9b206124c10 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLoggerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/logger/GridTestLog4jLoggerSelfTest.java
@@ -17,8 +17,8 @@
package org.apache.ignite.testframework.junits.logger;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.junit.AfterClass;
diff --git a/modules/core/src/test/java/org/apache/ignite/thread/ThreadPoolMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/thread/ThreadPoolMetricsTest.java
index 18132a0ff6e..178c128949f 100644
--- a/modules/core/src/test/java/org/apache/ignite/thread/ThreadPoolMetricsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/thread/ThreadPoolMetricsTest.java
@@ -40,6 +40,7 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.metric.MetricRegistry;
import org.apache.ignite.internal.processors.pool.PoolProcessor;
import org.apache.ignite.internal.util.StripedExecutor;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.plugin.AbstractTestPluginProvider;
@@ -189,7 +190,7 @@ public class ThreadPoolMetricsTest extends GridCommonAbstractTest {
int taskCnt = 10;
for (int i = 0; i < taskCnt; i++) {
- tasks.add(new GridTestUtils.IgniteRunnableX() {
+ tasks.add(new RunnableX() {
@Override public void runx() throws Exception {
U.sleep(1);
@@ -199,7 +200,7 @@ public class ThreadPoolMetricsTest extends GridCommonAbstractTest {
}
// Add a long task to check the metric of active tasks.
- tasks.add(new GridTestUtils.IgniteRunnableX() {
+ tasks.add(new RunnableX() {
@Override public void runx() throws Exception {
U.await(longTaskLatch, getTestTimeout(), MILLISECONDS);
}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java
index 59876b56687..788cf106f87 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicColumnsAbstractConcurrentSelfTest.java
@@ -56,13 +56,13 @@ import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
import org.apache.ignite.internal.processors.query.schema.message.SchemaFinishDiscoveryMessage;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgnitePredicate;
import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.GridTestUtils.RunnableX;
import org.junit.Test;
import static org.apache.ignite.internal.IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi;
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
index 6b4a5d145f8..f9f53bcf8b3 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractBasicSelfTest.java
@@ -36,6 +36,7 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.testframework.junits.WithSystemProperty;
import org.junit.Test;
@@ -45,7 +46,6 @@ import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
-import static org.apache.ignite.testframework.GridTestUtils.RunnableX;
import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
/**
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java
index ca423e7b809..b44fb44980a 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/DynamicIndexAbstractConcurrentSelfTest.java
@@ -48,6 +48,7 @@ import org.apache.ignite.internal.processors.query.QueryIndexDescriptorImpl;
import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
import org.apache.ignite.internal.processors.query.schema.SchemaIndexCacheVisitor;
import org.apache.ignite.internal.processors.query.schema.SchemaOperationException;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.T3;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -56,7 +57,6 @@ import org.jetbrains.annotations.NotNull;
import org.junit.Test;
import static org.apache.ignite.internal.IgniteClientReconnectAbstractTest.TestTcpDiscoverySpi;
-import static org.apache.ignite.testframework.GridTestUtils.RunnableX;
/**
* Concurrency tests for dynamic index create/drop.
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlFieldTypeValidationOnKeyValueInsertTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlFieldTypeValidationOnKeyValueInsertTest.java
index 619b1f17609..2eec5741232 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlFieldTypeValidationOnKeyValueInsertTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/SqlFieldTypeValidationOnKeyValueInsertTest.java
@@ -39,9 +39,9 @@ import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.index.AbstractIndexingCommonTest;
+import org.apache.ignite.internal.util.lang.RunnableX;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.testframework.GridTestUtils;
import org.junit.After;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -245,7 +245,7 @@ public class SqlFieldTypeValidationOnKeyValueInsertTest extends AbstractIndexing
}
/** */
- private void assertThrows(GridTestUtils.RunnableX runx, String msg) {
+ private void assertThrows(RunnableX runx, String msg) {
try {
runx.runx();
}