You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@ignite.apache.org by GitBox <gi...@apache.org> on 2019/01/11 09:17:41 UTC
[GitHub] asfgit closed pull request #5553: IGNITE-10205 add to utility command - ./control.sh --cache idle_verif…
asfgit closed pull request #5553: IGNITE-10205 add to utility command - ./control.sh --cache idle_verif…
URL: https://github.com/apache/ignite/pull/5553
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
index fbc881949ef0..bf596e76971e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/CommandHandler.java
@@ -197,9 +197,16 @@
/** */
private static final String CMD_SKIP_ZEROS = "--skip-zeros";
+ /** Command exclude caches. */
+ private static final String CMD_EXCLUDE_CACHES = "--excludeCaches";
+
/** Cache filter. */
private static final String CACHE_FILTER = "--cache-filter";
+ /** One cache filter option should used message. */
+ public static final String ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG = "Should use only one of option: " +
+ CMD_EXCLUDE_CACHES + ", " + CACHE_FILTER + " or pass caches explicitly";
+
/** */
private static final String CMD_USER_ATTRIBUTES = "--user-attributes";
@@ -859,7 +866,10 @@ private void printCacheHelp() {
usageCache(LIST, "regexPattern", op(or("groups", "seq")), OP_NODE_ID, op(CONFIG), op(OUTPUT_FORMAT, MULTI_LINE));
usageCache(CONTENTION, "minQueueSize", OP_NODE_ID, op("maxPrint"));
- usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), "[cache1,...,cacheN]", op(CACHE_FILTER, or(CacheFilterEnum.values())));
+ usageCache(IDLE_VERIFY, op(CMD_DUMP), op(CMD_SKIP_ZEROS), op(or(CMD_EXCLUDE_CACHES + " cache1,...,cacheN",
+ op(CACHE_FILTER, or(CacheFilterEnum.ALL.toString(), CacheFilterEnum.SYSTEM.toString(),
+ CacheFilterEnum.PERSISTENT.toString(), CacheFilterEnum.NOT_PERSISTENT.toString())), "cache1,...," +
+ "cacheN")));
usageCache(VALIDATE_INDEXES, "[cache1,...,cacheN]", OP_NODE_ID, op(or(VI_CHECK_FIRST + " N", VI_CHECK_THROUGH + " K")));
usageCache(DISTRIBUTION, or(NODE_ID, NULL), "[cacheName1,...,cacheNameN]", op(CMD_USER_ATTRIBUTES, "attrName1,...,attrNameN"));
usageCache(RESET_LOST_PARTITIONS, "cacheName1,...,cacheNameN");
@@ -1031,7 +1041,7 @@ else if (idleVerifyV2)
*/
private void legacyCacheIdleVerify(GridClient client, CacheArguments cacheArgs) throws GridClientException {
VisorIdleVerifyTaskResult res = executeTask(
- client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+ client, VisorIdleVerifyTask.class, new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches()));
Map<PartitionKey, List<PartitionHashRecord>> conflicts = res.getConflicts();
@@ -1180,7 +1190,8 @@ private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) th
String path = executeTask(
client,
VisorIdleVerifyDumpTask.class,
- new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.isSkipZeros(), cacheArgs.getCacheFilterEnum())
+ new VisorIdleVerifyDumpTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches(), cacheArgs.isSkipZeros(), cacheArgs
+ .getCacheFilterEnum())
);
log("VisorIdleVerifyDumpTask successfully written output to '" + path + "'");
@@ -1192,7 +1203,7 @@ private void cacheIdleVerifyDump(GridClient client, CacheArguments cacheArgs) th
*/
private void cacheIdleVerifyV2(GridClient client, CacheArguments cacheArgs) throws GridClientException {
IdleVerifyResultV2 res = executeTask(
- client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches()));
+ client, VisorIdleVerifyTaskV2.class, new VisorIdleVerifyTaskArg(cacheArgs.caches(), cacheArgs.excludeCaches()));
res.print(System.out::print);
}
@@ -2125,16 +2136,30 @@ private CacheArguments parseAndValidateCacheArgs() {
if (CMD_DUMP.equals(nextArg))
cacheArgs.dump(true);
+ else if (CMD_EXCLUDE_CACHES.equals(nextArg)) {
+ if (cacheArgs.caches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
+ throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
+ parseExcludeCacheNames(nextArg("Specify caches, which will be excluded."),
+ cacheArgs);
+ }
else if (CMD_SKIP_ZEROS.equals(nextArg))
cacheArgs.skipZeros(true);
else if (CACHE_FILTER.equals(nextArg)) {
+ if (cacheArgs.caches() != null || cacheArgs.excludeCaches() != null)
+ throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
String filter = nextArg("The cache filter should be specified. The following values can be " +
"used: " + Arrays.toString(CacheFilterEnum.values()) + '.');
cacheArgs.setCacheFilterEnum(CacheFilterEnum.valueOf(filter.toUpperCase()));
}
- else
+ else {
+ if (cacheArgs.excludeCaches() != null || cacheArgs.getCacheFilterEnum() != CacheFilterEnum.ALL)
+ throw new IllegalArgumentException(ONE_CACHE_FILTER_OPT_SHOULD_USED_MSG);
+
parseCacheNames(nextArg, cacheArgs);
+ }
}
break;
@@ -2312,6 +2337,24 @@ private void parseCacheNames(String cacheNames, CacheArguments cacheArgs) {
cacheArgs.caches(cacheNamesSet);
}
+ /**
+ * @param cacheNames Cache names arg.
+ * @param cacheArgs Cache args.
+ */
+ private void parseExcludeCacheNames(String cacheNames, CacheArguments cacheArgs) {
+ String[] cacheNamesArr = cacheNames.split(",");
+ Set<String> cacheNamesSet = new HashSet<>();
+
+ for (String cacheName : cacheNamesArr) {
+ if (F.isEmpty(cacheName))
+ throw new IllegalArgumentException("Non-empty cache names expected.");
+
+ cacheNamesSet.add(cacheName.trim());
+ }
+
+ cacheArgs.excludeCaches(cacheNamesSet);
+ }
+
/**
* Get ping param for grid client.
*
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
index 9372391397db..fb5bc88f4b1b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/cache/CacheArguments.java
@@ -33,6 +33,9 @@
/** Caches. */
private Set<String> caches;
+ /** Exclude caches or groups. */
+ private Set<String> excludeCaches;
+
/** Partition id. */
private int partId;
@@ -141,6 +144,20 @@ public void caches(Set<String> caches) {
this.caches = caches;
}
+ /**
+ * @return Exclude caches or groups.
+ */
+ public Set<String> excludeCaches() {
+ return excludeCaches;
+ }
+
+ /**
+ * @param excludeCaches Excluse caches or groups.
+ */
+ public void excludeCaches(Set<String> excludeCaches) {
+ this.excludeCaches = excludeCaches;
+ }
+
/**
* @return Partition id.
*/
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
index 5e872eb7c612..c30d37bdfea1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/VerifyBackupPartitionsTaskV2.java
@@ -50,6 +50,7 @@
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.topology.GridDhtPartitionState;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
@@ -205,9 +206,9 @@ public VerifyBackupPartitionsJobV2(VisorIdleVerifyTaskArg arg) {
@Override public Map<PartitionKeyV2, PartitionHashRecordV2> execute() throws IgniteException {
Set<Integer> grpIds = new HashSet<>();
- Set<String> missingCaches = new HashSet<>();
+ if (arg.getCaches() != null && !arg.getCaches().isEmpty()) {
+ Set<String> missingCaches = new HashSet<>();
- if (arg.getCaches() != null) {
for (String cacheName : arg.getCaches()) {
DynamicCacheDescriptor desc = ignite.context().cache().cacheDescriptor(cacheName);
@@ -229,14 +230,8 @@ else if (onlySpecificCaches()) {
grpIds.add(desc.groupId());
}
}
- else {
- Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
-
- for (CacheGroupContext grp : groups) {
- if (!grp.systemCache() && !grp.isLocal())
- grpIds.add(grp.groupId());
- }
- }
+ else
+ grpIds = getCacheGroupIds();
List<Future<Map<PartitionKeyV2, PartitionHashRecordV2>>> partHashCalcFutures = new ArrayList<>();
@@ -292,6 +287,45 @@ else if (e.getCause() instanceof IgniteException)
return res;
}
+ /**
+ * Gets filtered group ids.
+ */
+ private Set<Integer> getCacheGroupIds() {
+ Collection<CacheGroupContext> groups = ignite.context().cache().cacheGroups();
+
+ Set<Integer> grpIds = new HashSet<>();
+
+ if (arg.excludeCaches() == null || arg.excludeCaches().isEmpty()) {
+ for (CacheGroupContext grp : groups) {
+ if (!grp.systemCache() && !grp.isLocal())
+ grpIds.add(grp.groupId());
+ }
+ return grpIds;
+ }
+
+ for (CacheGroupContext grp : groups) {
+ if (!grp.systemCache() && !grp.isLocal() && !isGrpExcluded(grp))
+ grpIds.add(grp.groupId());
+ }
+
+ return grpIds;
+ }
+
+ /**
+ * @param grp Group.
+ */
+ private boolean isGrpExcluded(CacheGroupContext grp) {
+ if (arg.excludeCaches().contains(grp.name()))
+ return true;
+
+ for (GridCacheContext cacheCtx : grp.caches()) {
+ if (arg.excludeCaches().contains(cacheCtx.name()))
+ return true;
+ }
+
+ return false;
+ }
+
/**
* Checks and throw exception if caches was missed.
*
@@ -380,7 +414,6 @@ private boolean isCacheMatchFilter(String cacheName) {
});
}
-
/**
* @param grpCtx Group context.
* @param part Local partition.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
index 29dfb5b7a45b..3c836a40cdef 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyDumpTaskArg.java
@@ -45,11 +45,12 @@ public VisorIdleVerifyDumpTaskArg() {
/**
* @param caches Caches.
+ * @param excludeCaches Caches to exclude.
* @param skipZeros Skip zeros partitions.
* @param cacheFilterEnum Cache kind.
*/
- public VisorIdleVerifyDumpTaskArg(Set<String> caches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) {
- super(caches);
+ public VisorIdleVerifyDumpTaskArg(Set<String> caches, Set<String> excludeCaches, boolean skipZeros, CacheFilterEnum cacheFilterEnum) {
+ super(caches, excludeCaches);
this.skipZeros = skipZeros;
this.cacheFilterEnum = cacheFilterEnum;
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
index c82af5878ebb..d645fec16f7b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorIdleVerifyTaskArg.java
@@ -35,6 +35,9 @@
/** Caches. */
private Set<String> caches;
+ /** Exclude caches or groups. */
+ private Set<String> excludeCaches;
+
/**
* Default constructor.
*/
@@ -42,11 +45,21 @@ public VisorIdleVerifyTaskArg() {
// No-op.
}
+ /**
+ * @param caches Caches.
+ * @param excludeCaches Exclude caches or group.
+ */
+ public VisorIdleVerifyTaskArg(Set<String> caches, Set<String> excludeCaches) {
+ this.caches = caches;
+ this.excludeCaches = excludeCaches;
+ }
+
/**
* @param caches Caches.
*/
public VisorIdleVerifyTaskArg(Set<String> caches) {
this.caches = caches;
+ this.excludeCaches = excludeCaches;
}
@@ -57,14 +70,30 @@ public VisorIdleVerifyTaskArg(Set<String> caches) {
return caches;
}
+ /**
+ * @return Exclude caches or groups.
+ */
+ public Set<String> excludeCaches() {
+ return excludeCaches;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte getProtocolVersion() {
+ return V2;
+ }
+
/** {@inheritDoc} */
@Override protected void writeExternalData(ObjectOutput out) throws IOException {
U.writeCollection(out, caches);
+ U.writeCollection(out, excludeCaches);
}
/** {@inheritDoc} */
@Override protected void readExternalData(byte protoVer, ObjectInput in) throws IOException, ClassNotFoundException {
caches = U.readSet(in);
+
+ if (protoVer >= V2)
+ excludeCaches = U.readSet(in);
}
/** {@inheritDoc} */
diff --git a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
index 57ae628d06a8..e6a394902fc5 100644
--- a/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/util/GridCommandHandlerTest.java
@@ -1356,6 +1356,94 @@ public void testCacheIdleVerifyDumpForCorruptedDataOnNonePersistenceClientCache(
fail("Should be found dump with conflicts");
}
+ /**
+ * Tests that idle verify print partitions info with exclude cache group.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheIdleVerifyDumpExcludedCacheGrp() throws Exception {
+ IgniteEx ignite = (IgniteEx)startGrids(3);
+
+ ignite.cluster().active(true);
+
+ int parts = 32;
+
+ IgniteCache<Object, Object> cache = ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME));
+
+ IgniteCache<Object, Object> secondCache = ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "_second"));
+
+ injectTestSystemOut();
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--excludeCaches", "shared_grp"));
+
+ Matcher fileNameMatcher = dumpFileNameMatcher();
+
+ if (fileNameMatcher.find()) {
+ String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 0 partitions"));
+ }
+ else
+ fail("Should be found dump with conflicts");
+ }
+
+ /**
+ * Tests that idle verify print partitions info with exclude caches.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testCacheIdleVerifyDumpExcludedCaches() throws Exception {
+ IgniteEx ignite = (IgniteEx)startGrids(3);
+
+ ignite.cluster().active(true);
+
+ int parts = 32;
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME));
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setGroupName("shared_grp")
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "_second"));
+
+ ignite.createCache(new CacheConfiguration<>()
+ .setAffinity(new RendezvousAffinityFunction(false, parts))
+ .setBackups(1)
+ .setName(DEFAULT_CACHE_NAME + "_third"));
+
+ injectTestSystemOut();
+
+ assertEquals(EXIT_CODE_OK, execute("--cache", "idle_verify", "--dump", "--excludeCaches", DEFAULT_CACHE_NAME
+ + "," + DEFAULT_CACHE_NAME + "_second"));
+
+ Matcher fileNameMatcher = dumpFileNameMatcher();
+
+ if (fileNameMatcher.find()) {
+ String dumpWithConflicts = new String(Files.readAllBytes(Paths.get(fileNameMatcher.group(1))));
+
+ assertTrue(dumpWithConflicts.contains("idle_verify check has finished, found 32 partitions"));
+ assertTrue(dumpWithConflicts.contains("default_third"));
+ assertTrue(!dumpWithConflicts.contains("shared_grp"));
+ }
+ else
+ fail("Should be found dump with conflicts");
+ }
+
/**
* @return Build matcher for dump file name.
*/
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services