You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ir...@apache.org on 2019/01/11 09:14:45 UTC
[ignite] branch master updated: IGNITE-10205 Add to utility command
- ./control.sh --cache idle_verify --dump ability to exclude cache from
output file - Fixes #5553.
This is an automated email from the ASF dual-hosted git repository.
irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new d38f38f IGNITE-10205 Add to utility command - ./control.sh --cache idle_verify --dump ability to exclude cache from output file - Fixes #5553.
d38f38f is described below
commit d38f38f02958a927f541e88aac68d97a93af7340
Author: vd_pyatkov <vl...@gmail.com>
AuthorDate: Fri Jan 11 11:58:48 2019 +0300
IGNITE-10205 Add to utility command - ./control.sh --cache idle_verify --dump ability to exclude cache from output file - Fixes #5553.
Signed-off-by: Ivan Rakov <ir...@apache.org>
---
.../internal/commandline/CommandHandler.java | 53 +++++++++++--
.../internal/commandline/cache/CacheArguments.java | 17 +++++
.../cache/verify/VerifyBackupPartitionsTaskV2.java | 55 +++++++++++---
.../visor/verify/VisorIdleVerifyDumpTaskArg.java | 5 +-
.../visor/verify/VisorIdleVerifyTaskArg.java | 29 +++++++
.../apache/ignite/util/GridCommandHandlerTest.java | 88 ++++++++++++++++++++++
6 files changed, 229 insertions(+), 18 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index fbc8819..bf596e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -197,9 +197,16 @@ public class CommandHandler {
/** */
private static final String CMD_SKIP_ZEROS = "--skip-zeros";
+ /** Command exclude caches. */
+ private static final String CMD_EXCLUDE_CACHES = "--excludeCaches";
+
/** Cache filter. */
private static final String CACHE_FILTER = "--cache-filter";
+ /** One cache filter option should used message. */
+ public static final String ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG = "Should use only one of option: " +
+ CMD_EXCLUDE_CACHES + ", " + CACHE_FILTER + " or pass caches explicitly";
+
/** */
private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
@@ -859,7 +866,10 @@ public class CommandHandler {
usageCache(LIST, "regexPattern", op(or("groups", "seq")), OP_NODE_ID, op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE));
usageCache(CONTENTION, "minQueueSize", OP_NODE_ID, op("maxPrint"));
- usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", op(CACHE_FILTER, or(CacheFilterEnum.values())));
+ usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), op(or(CMD_EXCLUDE_CACHES + " cache1,...,cacheN",
+ op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(),
+ CacheFilterEnum.PERSISTENT.toString(), CacheFilterEnum.NOT_PERSISTENT.toString())), "cache1,...," +
+ "cacheN")));
usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", OP_NODE_ID, op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K")));
usageCache(DISTRIBUTION, or(NODE_ID, NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attrName1,...,attrNameN"));
usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN");
@@ -1031,7 +1041,7 @@ public class CommandHandler {
*/
private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws GridClientException {
VisorIdleVerifyTaskResult res = executeTask(
- client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+ client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches()));
Map<PartitionKey, List<PartitionHashRecord>> conflicts = res.getConflicts();
@@ -1180,7 +1190,8 @@ public class CommandHandler {
String path = executeTask(
client,
VisorIdleVerifyDumpTask.class,
- new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros(), cacheArgs.getCacheFilterEnum())
+ new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches(), cacheArgs.isSkipZeros(), cacheArgs
+ .getCacheFilterEnum())
);
log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'");
@@ -1192,7 +1203,7 @@ public class CommandHandler {
*/
private void cacheIdleVerifyV2(GridClient client, CacheArguments cacheArgs) throws GridClientException {
IdleVerifyResultV2 res = executeTask(
- client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+ client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches()));
res.print(System.out::print);
}
@@ -2125,16 +2136,30 @@ public class CommandHandler {
if (CMD_DUMP.equals(nextArg))
cacheArgs.dump(true);
+ else if (CMD_EXCLUDE_CACHES.equals(nextArg)) {
+ if (cacheArgs.caches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
+ throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
+ parseExcludeCacheNames(nextArg("Specify caches, which will be excluded."),
+ cacheArgs);
+ }
else if (CMD_SKIP_ZEROS.equals(nextArg))
cacheArgs.skipZeros(true);
else if (CACHE_FILTER.equals(nextArg)) {
+ if (cacheArgs.caches() != null || cacheArgs.excludeCaches() != null)
+ throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
String filter = nextArg("The cache filter should be specified. The following values can be " +
"used: " + Arrays.toString(CacheFilterEnum.values()) + '.');
cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter.toUpperCase()));
}
- else
+ else {
+ if (cacheArgs.excludeCaches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
+ throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
parseCacheNames(nextArg, cacheArgs);
+ }
}
break;
@@ -2313,6 +2338,24 @@ public class CommandHandler {
}
/**
+ * @param cacheNames Cache names arg.
+ * @param cacheArgs Cache args.
+ */
+ private void parseExcludeCacheNames(String cacheNames, CacheArguments cacheArgs) {
+ String[] cacheNamesArr = cacheNames.split(",");
+ Set<String> cacheNamesSet = new HashSet<>();
+
+ for (String cacheName : cacheNamesArr) {
+ if (F.isEmpty(cacheName))
+ throw new IllegalArgumentException("Non-empty cache names expected.");
+
+ cacheNamesSet.add(cacheName.trim());
+ }
+
+ cacheArgs.excludeCaches(cacheNamesSet);
+ }
+
+ /**
* Get ping param for grid client.
*
* @param nextArgErr Argument extraction error message.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 9372391..fb5bc88 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -33,6 +33,9 @@ public class CacheArguments {
/** Caches. */
private Set<String> caches;
+ /** Exclude caches or groups. */
+ private Set<String> excludeCaches;
+
/** Partition id. */
private int partId;
@@ -142,6 +145,20 @@ public class CacheArguments {
}
/**
+ * @return Exclude caches or groups.
+ */
+ public Set<String> excludeCaches() {
+ return excludeCaches;
+ }
+
+ /**
+ * @param excludeCaches Excluse caches or groups.
+ */
+ public void excludeCaches(Set<String> excludeCaches) {
+ this.excludeCaches = excludeCaches;
+ }
+
+ /**
* @return Partition id.
*/
public int partitionId() {
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 5e872eb..c30d37b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -50,6 +50,7 @@ import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -205,9 +206,9 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
@Override public Map<PartitionKeyV2, PartitionHashRecordV2> execute() throws IgniteException {
Set<Integer> grpIds = new HashSet<>();
- Set<String> missingCaches = new HashSet<>();
+ if (arg.getCaches() != null && !arg.getCaches().isEmpty()) {
+ Set<String> missingCaches = new HashSet<>();
- if (arg.getCaches() != null) {
for (String cacheName : arg.getCaches()) {
DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
@@ -229,14 +230,8 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
grpIds.add(desc.groupId());
}
}
- else {
- Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
-
- for (CacheGroupContext grp : groups) {
- if (!grp.systemCache() && !grp.isLocal())
- grpIds.add(grp.groupId());
- }
- }
+ else
+ grpIds = getCacheGroupIds();
List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFutures = new ArrayList<>();
@@ -293,6 +288,45 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
}
/**
+ * Gets filtered group ids.
+ */
+ private Set<Integer> getCacheGroupIds() {
+ Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+ Set<Integer> grpIds = new HashSet<>();
+
+ if (arg.excludeCaches() == null || arg.excludeCaches().isEmpty()) {
+ for (CacheGroupContext grp : groups) {
+ if (!grp.systemCache() && !grp.isLocal())
+ grpIds.add(grp.groupId());
+ }
+ return grpIds;
+ }
+
+ for (CacheGroupContext grp : groups) {
+ if (!grp.systemCache() && !grp.isLocal() && !isGrpExcluded(grp))
+ grpIds.add(grp.groupId());
+ }
+
+ return grpIds;
+ }
+
+ /**
+ * @param grp Group.
+ */
+ private boolean isGrpExcluded(CacheGroupContext grp) {
+ if (arg.excludeCaches().contains(grp.name()))
+ return true;
+
+ for (GridCacheContext cacheCtx : grp.caches()) {
+ if (arg.excludeCaches().contains(cacheCtx.name()))
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
* Checks and throw exception if caches was missed.
*
* @param missingCaches Missing caches.
@@ -380,7 +414,6 @@ public class VerifyBackupPartitionsTaskV2 extends ComputeTaskAdapter<VisorIdleVe
});
}
-
/**
* @param grpCtx Group context.
* @param part Local partition.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
index 29dfb5b..3c836a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
@@ -45,11 +45,12 @@ public class VisorIdleVerifyDumpTaskArg extends VisorIdleVerifyTaskArg {
/**
* @param caches Caches.
+ * @param excludeCaches Caches to exclude.
* @param skipZeros Skip zeros partitions.
* @param cacheFilterEnum Cache kind.
*/
- public VisorIdleVerifyDumpTaskArg(Set<String> caches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) {
- super(caches);
+ public VisorIdleVerifyDumpTaskArg(Set<String> caches, Set<String> excludeCaches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) {
+ super(caches, excludeCaches);
this.skipZeros = skipZeros;
this.cacheFilterEnum = cacheFilterEnum;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
index c82af58..d645fec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
@@ -35,6 +35,9 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
/** Caches. */
private Set<String> caches;
+ /** Exclude caches or groups. */
+ private Set<String> excludeCaches;
+
/**
* Default constructor.
*/
@@ -44,9 +47,19 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
/**
* @param caches Caches.
+ * @param excludeCaches Exclude caches or group.
+ */
+ public VisorIdleVerifyTaskArg(Set<String> caches, Set<String> excludeCaches) {
+ this.caches = caches;
+ this.excludeCaches = excludeCaches;
+ }
+
+ /**
+ * @param caches Caches.
*/
public VisorIdleVerifyTaskArg(Set<String> caches) {
this.caches = caches;
+ this.excludeCaches = excludeCaches;
}
@@ -57,14 +70,30 @@ public class VisorIdleVerifyTaskArg extends VisorDataTransferObject {
return caches;
}
+ /**
+ * @return Exclude caches or groups.
+ */
+ public Set<String> excludeCaches() {
+ return excludeCaches;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte getProtocolVersion() {
+ return V2;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
U.writeCollection(out, caches);
+ U.writeCollection(out, excludeCaches);
}
/** {@inheritDoc} */
@Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
caches = U.readSet(in);
+
+ if (protoVer >= V2)
+ excludeCaches = U.readSet(in);
}
/** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 57ae628..e6a3949 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -1357,6 +1357,94 @@ public class GridCommandHandlerTest extends GridCommonAbstractTest {
}
/**
+ * Tests that idle verify print partitions info with exclude cache group.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheIdleVerifyDumpExcludedCacheGrp() throws Exception {
+ IgniteEx ignite = (IgniteEx)startGrids(3);
+
+ ignite.cluster().active(true);
+
+ int parts = 32;
+
+ IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME));
+
+ IgniteCache<Object, Object> secondCache = ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "_second"));
+
+ injectTestSystemOut();
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--excludeCaches", "shared_grp"));
+
+ Matcher fileNameMatcher = dumpFileNameMatcher();
+
+ if (fileNameMatcher.find()) {
+ String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 0 partitions"));
+ }
+ else
+ fail("Should be found dump with conflicts");
+ }
+
+ /**
+ * Tests that idle verify print partitions info with exclude caches.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheIdleVerifyDumpExcludedCaches() throws Exception {
+ IgniteEx ignite = (IgniteEx)startGrids(3);
+
+ ignite.cluster().active(true);
+
+ int parts = 32;
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME));
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "_second"));
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "_third"));
+
+ injectTestSystemOut();
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--excludeCaches", DEFAULT_CACHE_NAME
+ + "," + DEFAULT_CACHE_NAME + "_second"));
+
+ Matcher fileNameMatcher = dumpFileNameMatcher();
+
+ if (fileNameMatcher.find()) {
+ String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 32 partitions"));
+ assertTrue(dumpWithConflicts.contains("default_third"));
+ assertTrue(!dumpWithConflicts.contains("shared_grp"));
+ }
+ else
+ fail("Should be found dump with conflicts");
+ }
+
+ /**
* @return Build matcher for dump file name.
*/
@NotNull private Matcher dumpFileNameMatcher() {