You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ir...@apache.org on 2018/05/24 15:13:27 UTC
ignite git commit: IGNITE-8560 Update index validation utility to use
statistical check approach - Fixes #4051.
Repository: ignite
Updated Branches:
refs/heads/master 86c18998d -> 76e1fe754
IGNITE-8560 Update index validation utility to use statistical check approach - Fixes #4051.
Signed-off-by: Ivan Rakov <ir...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/76e1fe75
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/76e1fe75
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/76e1fe75
Branch: refs/heads/master
Commit: 76e1fe754d7a8bd059d7ab64d17cbefa4913a702
Parents: 86c1899
Author: Sergey Chugunov <se...@gmail.com>
Authored: Thu May 24 18:11:47 2018 +0300
Committer: Ivan Rakov <ir...@apache.org>
Committed: Thu May 24 18:11:47 2018 +0300
----------------------------------------------------------------------
.../internal/commandline/CommandHandler.java | 92 +++++++++++++++-----
.../commandline/cache/CacheArguments.java | 34 ++++++++
.../verify/VisorValidateIndexesTaskArg.java | 40 ++++++++-
.../commandline/CommandHandlerParsingTest.java | 87 ++++++++++++++++++
.../visor/verify/ValidateIndexesClosure.java | 66 +++++++++++++-
.../visor/verify/VisorValidateIndexesTask.java | 2 +-
.../util/GridCommandHandlerIndexingTest.java | 8 +-
7 files changed, 305 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index 47cc233..c59e348 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -181,6 +181,12 @@ public class CommandHandler {
/** */
private static final String BASELINE_SET_VERSION = "version";
+ /** Parameter name for validate_indexes command. */
+ static final String VI_CHECK_FIRST = "checkFirst";
+
+ /** Parameter name for validate_indexes command. */
+ static final String VI_CHECK_THROUGH = "checkThrough";
+
/** */
static final String WAL_PRINT = "print";
@@ -583,10 +589,12 @@ public class CommandHandler {
usage(" Show information about caches, groups or sequences that match a regex:", CACHE, " list regexPattern [groups|seq] [nodeId]");
usage(" Show hot keys that are point of contention for multiple transactions:", CACHE, " contention minQueueSize [nodeId] [maxPrint]");
usage(" Verify partition counters and hashes between primary and backups on idle cluster:", CACHE, " idle_verify [cache1,...,cacheN]");
- usage(" Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId]");
+ usage(" Validate custom indexes on idle cluster:", CACHE, " validate_indexes [cache1,...,cacheN] [nodeId] [checkFirst|checkThrough]");
- log(" If [nodeId] is not specified, cont and validate_indexes commands will be broadcasted to all server nodes.");
+ log(" If [nodeId] is not specified, contention and validate_indexes commands will be broadcasted to all server nodes.");
log(" Another commands where [nodeId] is optional will run on a random server node.");
+ log(" checkFirst numeric parameter for validate_indexes specifies number of first K keys to be validated.");
+ log(" checkThrough numeric parameter for validate_indexes allows to check each Kth key.");
nl();
}
@@ -624,7 +632,11 @@ public class CommandHandler {
* @param cacheArgs Cache args.
*/
private void cacheValidateIndexes(GridClient client, CacheArguments cacheArgs) throws GridClientException {
- VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(cacheArgs.caches());
+ VisorValidateIndexesTaskArg taskArg = new VisorValidateIndexesTaskArg(
+ cacheArgs.caches(),
+ cacheArgs.checkFirst(),
+ cacheArgs.checkThrough()
+ );
UUID nodeId = cacheArgs.nodeId() == null ? BROADCAST_UUID : cacheArgs.nodeId();
@@ -1407,7 +1419,8 @@ public class CommandHandler {
break;
case IDLE_VERIFY:
- parseCacheNamesIfPresent(cacheArgs);
+ if (hasNextCacheArg())
+ parseCacheNames(nextArg(""), cacheArgs);
break;
@@ -1425,10 +1438,53 @@ public class CommandHandler {
break;
case VALIDATE_INDEXES:
- parseCacheNamesIfPresent(cacheArgs);
+ int argsCnt = 0;
- if (hasNextCacheArg())
- cacheArgs.nodeId(UUID.fromString(nextArg("")));
+ while (hasNextCacheArg() && argsCnt++ < 4) {
+ String arg = nextArg("");
+
+ if (VI_CHECK_FIRST.equals(arg) || VI_CHECK_THROUGH.equals(arg)) {
+ if (!hasNextCacheArg())
+ throw new IllegalArgumentException("Numeric value for '" + arg + "' parameter expected.");
+
+ int numVal;
+
+ String numStr = nextArg("");
+
+ try {
+ numVal = Integer.parseInt(numStr);
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "Not numeric value was passed for '"
+ + arg
+ + "' parameter: "
+ + numStr
+ );
+ }
+
+ if (numVal <= 0)
+ throw new IllegalArgumentException("Value for '" + arg + "' property should be positive.");
+
+ if (VI_CHECK_FIRST.equals(arg))
+ cacheArgs.checkFirst(numVal);
+ else
+ cacheArgs.checkThrough(numVal);
+
+ continue;
+ }
+
+ try {
+ cacheArgs.nodeId(UUID.fromString(arg));
+
+ continue;
+ }
+ catch (IllegalArgumentException ignored) {
+ //No-op.
+ }
+
+ parseCacheNames(arg, cacheArgs);
+ }
break;
@@ -1473,22 +1529,18 @@ public class CommandHandler {
/**
* @param cacheArgs Cache args.
*/
- private void parseCacheNamesIfPresent(CacheArguments cacheArgs) {
- if (hasNextCacheArg()) {
- String cacheNames = nextArg("");
-
- String[] cacheNamesArr = cacheNames.split(",");
- Set<String> cacheNamesSet = new HashSet<>();
+ private void parseCacheNames(String cacheNames, CacheArguments cacheArgs) {
+ String[] cacheNamesArr = cacheNames.split(",");
+ Set<String> cacheNamesSet = new HashSet<>();
- for (String cacheName : cacheNamesArr) {
- if (F.isEmpty(cacheName))
- throw new IllegalArgumentException("Non-empty cache names expected.");
+ for (String cacheName : cacheNamesArr) {
+ if (F.isEmpty(cacheName))
+ throw new IllegalArgumentException("Non-empty cache names expected.");
- cacheNamesSet.add(cacheName.trim());
- }
-
- cacheArgs.caches(cacheNamesSet);
+ cacheNamesSet.add(cacheName.trim());
}
+
+ cacheArgs.caches(cacheNamesSet);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 6f315ef..1411b2a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -46,6 +46,12 @@ public class CacheArguments {
/** Max print. */
private int maxPrint;
+ /** validate_indexes 'checkFirst' argument */
+ private int checkFirst = -1;
+
+ /** validate_indexes 'checkThrough' argument */
+ private int checkThrough = -1;
+
/** Cache view command. */
private @Nullable VisorViewCacheCmd cacheCmd;
@@ -160,4 +166,32 @@ public class CacheArguments {
public void maxPrint(int maxPrint) {
this.maxPrint = maxPrint;
}
+
+ /**
+ * @return Max number of entries to be checked.
+ */
+ public int checkFirst() {
+ return checkFirst;
+ }
+
+ /**
+ * @param checkFirst Max number of entries to be checked.
+ */
+ public void checkFirst(int checkFirst) {
+ this.checkFirst = checkFirst;
+ }
+
+ /**
+ * @return Number of entries to check through.
+ */
+ public int checkThrough() {
+ return checkThrough;
+ }
+
+ /**
+ * @param checkThrough Number of entries to check through.
+ */
+ public void checkThrough(int checkThrough) {
+ this.checkThrough = checkThrough;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
index cf9aff5..aa49977 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTaskArg.java
@@ -35,6 +35,12 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
/** Caches. */
private Set<String> caches;
+ /** Check first K elements. */
+ private int checkFirst;
+
+ /** Check through K element (skip K-1, check Kth). */
+ private int checkThrough;
+
/**
* Default constructor.
*/
@@ -45,8 +51,10 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
/**
* @param caches Caches.
*/
- public VisorValidateIndexesTaskArg(Set<String> caches) {
+ public VisorValidateIndexesTaskArg(Set<String> caches, int checkFirst, int checkThrough) {
this.caches = caches;
+ this.checkFirst = checkFirst;
+ this.checkThrough = checkThrough;
}
@@ -57,14 +65,44 @@ public class VisorValidateIndexesTaskArg extends VisorDataTransferObject {
return caches;
}
+ /**
+ * @return checkFirst.
+ */
+ public int getCheckFirst() {
+ return checkFirst;
+ }
+
+ /**
+ * @return checkThrough.
+ */
+ public int getCheckThrough() {
+ return checkThrough;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
U.writeCollection(out, caches);
+ out.writeInt(checkFirst);
+ out.writeInt(checkThrough);
}
/** {@inheritDoc} */
@Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
caches = U.readSet(in);
+
+ if (protoVer > V1) {
+ checkFirst = in.readInt();
+ checkThrough = in.readInt();
+ }
+ else {
+ checkFirst = -1;
+ checkThrough = -1;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte getProtocolVersion() {
+ return V2;
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
index 2fc40ca..737c0c7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/commandline/CommandHandlerParsingTest.java
@@ -21,15 +21,20 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.UUID;
import junit.framework.TestCase;
+import org.apache.ignite.internal.commandline.cache.CacheArguments;
+import org.apache.ignite.internal.commandline.cache.CacheCommand;
import org.apache.ignite.internal.visor.tx.VisorTxProjection;
import org.apache.ignite.internal.visor.tx.VisorTxSortOrder;
import org.apache.ignite.internal.visor.tx.VisorTxTaskArg;
import static java.util.Arrays.asList;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_ENABLE_EXPERIMENTAL_COMMAND;
+import static org.apache.ignite.internal.commandline.Command.CACHE;
import static org.apache.ignite.internal.commandline.Command.WAL;
import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_HOST;
import static org.apache.ignite.internal.commandline.CommandHandler.DFLT_PORT;
+import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_FIRST;
+import static org.apache.ignite.internal.commandline.CommandHandler.VI_CHECK_THROUGH;
import static org.apache.ignite.internal.commandline.CommandHandler.WAL_DELETE;
import static org.apache.ignite.internal.commandline.CommandHandler.WAL_PRINT;
@@ -52,6 +57,88 @@ public class CommandHandlerParsingTest extends TestCase {
}
/**
+ * validate_indexes command arguments parsing and validation
+ */
+ public void testValidateIndexArguments() {
+ CommandHandler hnd = new CommandHandler();
+
+ //happy case for all parameters
+ try {
+ int expectedCheckFirst = 10;
+ int expectedCheckThrough = 11;
+ UUID nodeId = UUID.randomUUID();
+
+ CacheArguments args = hnd.parseAndValidate(
+ Arrays.asList(
+ CACHE.text(),
+ CacheCommand.VALIDATE_INDEXES.text(),
+ "cache1, cache2",
+ nodeId.toString(),
+ VI_CHECK_FIRST,
+ Integer.toString(expectedCheckFirst),
+ VI_CHECK_THROUGH,
+ Integer.toString(expectedCheckThrough)
+ )
+ ).cacheArgs();
+
+ assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId());
+ assertEquals("checkFirst parameter unexpected value", expectedCheckFirst, args.checkFirst());
+ assertEquals("checkThrough parameter unexpected value", expectedCheckThrough, args.checkThrough());
+ }
+ catch (IllegalArgumentException e) {
+ fail("Unexpected exception: " + e);
+ }
+
+ try {
+ int expectedParam = 11;
+ UUID nodeId = UUID.randomUUID();
+
+ CacheArguments args = hnd.parseAndValidate(
+ Arrays.asList(
+ CACHE.text(),
+ CacheCommand.VALIDATE_INDEXES.text(),
+ nodeId.toString(),
+ VI_CHECK_THROUGH,
+ Integer.toString(expectedParam)
+ )
+ ).cacheArgs();
+
+ assertNull("caches weren't specified, null value expected", args.caches());
+ assertEquals("nodeId parameter unexpected value", nodeId, args.nodeId());
+ assertEquals("checkFirst parameter unexpected value", -1, args.checkFirst());
+ assertEquals("checkThrough parameter unexpected value", expectedParam, args.checkThrough());
+ }
+ catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ hnd.parseAndValidate(
+ Arrays.asList(
+ CACHE.text(),
+ CacheCommand.VALIDATE_INDEXES.text(),
+ VI_CHECK_FIRST,
+ "0"
+ )
+ );
+
+ fail("Expected exception hasn't been thrown");
+ }
+ catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ }
+
+ try {
+ hnd.parseAndValidate(Arrays.asList(CACHE.text(), CacheCommand.VALIDATE_INDEXES.text(), VI_CHECK_THROUGH));
+
+ fail("Expected exception hasn't been thrown");
+ }
+ catch (IllegalArgumentException e) {
+ e.printStackTrace();
+ }
+ }
+
+ /**
* Test that experimental command (i.e. WAL command) is disabled by default.
*/
public void testExperimentalCommandIsDisabled() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index e0eff61..e01dca2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -89,6 +89,12 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
/** Cache names. */
private Set<String> cacheNames;
+ /** If provided only first K elements will be validated. */
+ private final int checkFirst;
+
+ /** If provided only each Kth element will be validated. */
+ private final int checkThrough;
+
/** Counter of processed partitions. */
private final AtomicInteger processedPartitions = new AtomicInteger(0);
@@ -109,9 +115,13 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
/**
* @param cacheNames Cache names.
+ * @param checkFirst If positive only first K elements will be validated.
+ * @param checkThrough If positive only each Kth element will be validated.
*/
- public ValidateIndexesClosure(Set<String> cacheNames) {
+ public ValidateIndexesClosure(Set<String> cacheNames, int checkFirst, int checkThrough) {
this.cacheNames = cacheNames;
+ this.checkFirst = checkFirst;
+ this.checkThrough = checkThrough;
}
/** {@inheritDoc} */
@@ -320,12 +330,39 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
m.setAccessible(true);
+ final boolean skipConditions = checkFirst > 0 || checkThrough > 0;
+ final boolean bothSkipConditions = checkFirst > 0 && checkThrough > 0;
+
+ long current = 0;
+ long processedNumber = 0;
+
while (it.hasNextX()) {
if (enoughIssues)
break;
CacheDataRow row = it.nextX();
+ if (skipConditions) {
+ if (bothSkipConditions) {
+ if (processedNumber > checkFirst)
+ break;
+ else if (current++ % checkThrough > 0)
+ continue;
+ else
+ processedNumber++;
+ }
+ else {
+ if (checkFirst > 0) {
+ if (current++ > checkFirst)
+ break;
+ }
+ else {
+ if (current++ % checkThrough > 0)
+ continue;
+ }
+ }
+ }
+
int cacheId = row.cacheId() == 0 ? grpCtx.groupId() : row.cacheId();
GridCacheContext cacheCtx = row.cacheId() == 0 ?
@@ -462,6 +499,12 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
enoughIssues = true;
}
+ final boolean skipConditions = checkFirst > 0 || checkThrough > 0;
+ final boolean bothSkipConditions = checkFirst > 0 && checkThrough > 0;
+
+ long current = 0;
+ long processedNumber = 0;
+
while (!enoughIssues) {
KeyCacheObject h2key = null;
@@ -471,6 +514,27 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
GridH2Row h2Row = (GridH2Row)cursor.get();
+ if (skipConditions) {
+ if (bothSkipConditions) {
+ if (processedNumber > checkFirst)
+ break;
+ else if (current++ % checkThrough > 0)
+ continue;
+ else
+ processedNumber++;
+ }
+ else {
+ if (checkFirst > 0) {
+ if (current++ > checkFirst)
+ break;
+ }
+ else {
+ if (current++ % checkThrough > 0)
+ continue;
+ }
+ }
+ }
+
h2key = h2Row.key();
CacheDataRow cacheDataStoreRow = ctx.group().offheap().read(ctx, h2key);
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
index 52b48a5..abb7f7e 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesTask.java
@@ -76,7 +76,7 @@ public class VisorValidateIndexesTask extends VisorMultiNodeTask<VisorValidateIn
/** {@inheritDoc} */
@Override protected VisorValidateIndexesJobResult run(@Nullable VisorValidateIndexesTaskArg arg) throws IgniteException {
try {
- ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches());
+ ValidateIndexesClosure clo = new ValidateIndexesClosure(arg.getCaches(), arg.getCheckFirst(), arg.getCheckThrough());
ignite.context().resource().injectGeneric(clo);
http://git-wip-us.apache.org/repos/asf/ignite/blob/76e1fe75/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
index 62d3fc0..ca9aa53 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexingTest.java
@@ -101,7 +101,13 @@ public class GridCommandHandlerIndexingTest extends GridCommandHandlerTest {
injectTestSystemOut();
- assertEquals(EXIT_CODE_OK, execute("--cache", "validate_indexes", cacheName));
+ assertEquals(EXIT_CODE_OK,
+ execute(
+ "--cache",
+ "validate_indexes",
+ cacheName,
+ "checkFirst", "10000",
+ "checkThrough", "10"));
assertTrue(testOut.toString().contains("validate_indexes has finished with errors"));
}