You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by se...@apache.org on 2021/05/13 06:46:55 UTC
[ignite] branch master updated: IGNITE-14469 Additional tests for
control utility command indexes_force_rebuild - Fixes #9061.
This is an automated email from the ASF dual-hosted git repository.
sergeychugunov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 29cd623 IGNITE-14469 Additional tests for control utility command indexes_force_rebuild - Fixes #9061.
29cd623 is described below
commit 29cd6233e6d4f058e4714d36d5d04d9ed4f5a8c6
Author: Eduard Rakhmankulov <er...@gmail.com>
AuthorDate: Thu May 13 09:45:35 2021 +0300
IGNITE-14469 Additional tests for control utility command indexes_force_rebuild - Fixes #9061.
Signed-off-by: Sergey Chugunov <se...@gmail.com>
---
.../cache/CacheIndexesForceRebuild.java | 9 +-
.../GridCommandHandlerIndexForceRebuildTest.java | 172 ++++++++++++++++++---
.../processors/cache/GridCacheProcessor.java | 3 +-
.../visor/cache/index/IndexForceRebuildTask.java | 135 +++++-----------
4 files changed, 195 insertions(+), 124 deletions(-)
diff --git a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java
index 8901277..91c2fb5 100644
--- a/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java
+++ b/modules/control-utility/src/main/java/org/apache/ignite/internal/commandline/cache/CacheIndexesForceRebuild.java
@@ -84,8 +84,13 @@ public class CacheIndexesForceRebuild extends AbstractCommand<CacheIndexesForceR
final UUID nodeId = args.nodeId;
try (GridClient client = Command.startClient(clientCfg)) {
- taskRes = TaskExecutor.executeTaskByNameOnNode(client, IndexForceRebuildTask.class.getName(), taskArg,
- nodeId, clientCfg);
+ taskRes = TaskExecutor.executeTaskByNameOnNode(
+ client,
+ IndexForceRebuildTask.class.getName(),
+ taskArg,
+ nodeId,
+ clientCfg
+ );
}
printResult(taskRes, logger);
diff --git a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java
index 5cad551..80de9ab 100644
--- a/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java
+++ b/modules/control-utility/src/test/java/org/apache/ignite/util/GridCommandHandlerIndexForceRebuildTest.java
@@ -40,6 +40,8 @@ import org.apache.ignite.internal.processors.query.schema.SchemaIndexOperationCa
import org.apache.ignite.internal.util.future.GridFutureAdapter;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.ListeningTestLogger;
import org.apache.ignite.testframework.LogListener;
@@ -51,6 +53,7 @@ import org.junit.Test;
import static java.lang.String.valueOf;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_INVALID_ARGUMENTS;
import static org.apache.ignite.internal.commandline.CommandHandler.EXIT_CODE_OK;
+import static org.apache.ignite.internal.commandline.CommandLogger.INDENT;
import static org.apache.ignite.testframework.GridTestUtils.assertContains;
import static org.apache.ignite.testframework.GridTestUtils.getFieldValue;
import static org.apache.ignite.testframework.GridTestUtils.runAsync;
@@ -228,7 +231,15 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA
waitForIndexesRebuild(grid(LAST_NODE_NUM));
- validateTestCacheNamesArgOutput();
+ String outputStr = testOut.toString();
+
+ validateOutputCacheNamesNotFound(outputStr, CACHE_NAME_NON_EXISTING);
+
+ validateOutputIndicesRebuildingInProgress(outputStr, F.asMap(GRP_NAME_2, F.asList(CACHE_NAME_2_1)));
+
+ validateOutputIndicesRebuildWasStarted(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_1)));
+
+ assertEquals("Unexpected number of lines in output.", 19, outputStr.split("\n").length);
// Index rebuild must be triggered only for cache1_1 and only on node3.
assertFalse(cache1Listeners[0].check());
@@ -279,7 +290,21 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA
waitForIndexesRebuild(grid(LAST_NODE_NUM));
- validateTestCacheGroupArgOutput();
+ String outputStr = testOut.toString();
+
+ validateOutputCacheGroupsNotFound(outputStr, GRP_NAME_NON_EXISTING);
+
+ validateOutputIndicesRebuildingInProgress(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_2)));
+
+ validateOutputIndicesRebuildWasStarted(
+ outputStr,
+ F.asMap(
+ GRP_NAME_1, F.asList(CACHE_NAME_1_1),
+ GRP_NAME_2, F.asList(CACHE_NAME_2_1)
+ )
+ );
+
+ assertEquals("Unexpected number of lines in outputStr.", 20, outputStr.split("\n").length);
assertFalse(cache1Listeners[0].check());
assertFalse(cache1Listeners[1].check());
@@ -476,40 +501,119 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA
}
/**
- * Validated control.sh utility output for {@link #testCacheNamesArg()}.
+ * Checking that a sequence of forced rebuild of indexes is possible
+ *
+ * @throws Exception If failed.
*/
- private void validateTestCacheNamesArgOutput() {
- String outputStr = testOut.toString();
+ @Test
+ public void testSequentialForceRebuildIndexes() throws Exception {
+ IgniteEx grid = grid(0);
+
+ injectTestSystemOut();
+
+ String outputStr;
- assertTrue(outputStr.contains("WARNING: These caches were not found:\n" +
- " " + CACHE_NAME_NON_EXISTING));
+ forceRebuildIndices(F.asList(CACHE_NAME_1_1), grid);
- assertTrue(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:\n" +
- " groupName=" + GRP_NAME_2 + ", cacheName=" + CACHE_NAME_2_1));
+ outputStr = testOut.toString();
- assertTrue(outputStr.contains("Indexes rebuild was started for these caches:\n" +
- " groupName=" + GRP_NAME_1 + ", cacheName=" + CACHE_NAME_1_1));
+ validateOutputIndicesRebuildWasStarted(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_1)));
+
+ assertFalse(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:"));
+
+ forceRebuildIndices(F.asList(CACHE_NAME_1_1), grid);
+
+ validateOutputIndicesRebuildWasStarted(outputStr, F.asMap(GRP_NAME_1, F.asList(CACHE_NAME_1_1)));
+
+ assertFalse(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:"));
+ }
+
+ /**
+ * Validates control.sh output when caches by name not found.
+ *
+ * @param outputStr CLI {@code control.sh} utility output.
+ * @param cacheNames Cache names to print.
+ */
+ private void validateOutputCacheNamesNotFound(String outputStr, String... cacheNames) {
+ assertContains(
+ log,
+ outputStr,
+ "WARNING: These caches were not found:" + U.nl() + makeStringListWithIndent(cacheNames)
+ );
+ }
+
+ /**
+ * Validates control.sh output when caches by group not found.
+ *
+ * @param outputStr CLI {@code control.sh} utility output.
+ * @param cacheGrps Cache groups to print.
+ */
+ private void validateOutputCacheGroupsNotFound(String outputStr, String... cacheGrps) {
+ assertContains(
+ log,
+ outputStr,
+ "WARNING: These cache groups were not found:" + U.nl() + makeStringListWithIndent(cacheGrps)
+ );
+ }
- assertEquals("Unexpected number of lines in output.", 19, outputStr.split("\n").length);
+ /**
+ * Makes new-line List with indent.
+ * @param strings List of strings.
+ * @return Formated text.
+ */
+ private String makeStringListWithIndent(String... strings) {
+ return INDENT + String.join(U.nl() + INDENT, strings);
}
/**
- * Validated control.sh utility output for {@link #testGroupNamesArg()}.
+ * Makes formatted text for given caches.
+ *
+ * @param cacheGroputToNames Cache groups mapping to non-existing cache names.
+ * @return Text for CLI print output for given caches.
*/
- private void validateTestCacheGroupArgOutput() {
- String outputStr = testOut.toString();
+ private String makeStringListForCacheGroupsAndNames(Map<String, List<String>> cacheGroputToNames) {
+ SB sb = new SB();
+
+ for (Map.Entry<String, List<String>> entry : cacheGroputToNames.entrySet()) {
+ String cacheGrp = entry.getKey();
- assertTrue(outputStr.contains("WARNING: These cache groups were not found:\n" +
- " " + GRP_NAME_NON_EXISTING));
+ for (String cacheName : entry.getValue())
+ sb.a(INDENT).a("groupName=").a(cacheGrp).a(", cacheName=").a(cacheName).a(U.nl());
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Validates control.sh output when some indices rebuilt in progress.
+ *
+ * @param outputStr CLI {@code control.sh} utility output.
+ * @param cacheGroputToNames Cache groups mapping to non-existing cache names.
+ */
+ private void validateOutputIndicesRebuildingInProgress(String outputStr, Map<String, List<String>> cacheGroputToNames) {
+ String caches = makeStringListForCacheGroupsAndNames(cacheGroputToNames);
- assertTrue(outputStr.contains("WARNING: These caches have indexes rebuilding in progress:\n" +
- " groupName=" + GRP_NAME_1 + ", cacheName=" + CACHE_NAME_1_2));
+ assertContains(
+ log,
+ outputStr,
+ "WARNING: These caches have indexes rebuilding in progress:" + U.nl() + caches
+ );
+ }
- assertTrue(outputStr.contains("Indexes rebuild was started for these caches:\n" +
- " groupName=" + GRP_NAME_1 + ", cacheName=" + CACHE_NAME_1_1 + "\n" +
- " groupName=" + GRP_NAME_2 + ", cacheName=" + CACHE_NAME_2_1));
+ /**
+ * Validates control.sh output when indices started to rebuild.
+ *
+ * @param outputStr CLI {@code control.sh} utility output.
+ * @param cacheGroputToNames Cache groups mapping to non-existing cache names.
+ */
+ private void validateOutputIndicesRebuildWasStarted(String outputStr, Map<String, List<String>> cacheGroputToNames) {
+ String caches = makeStringListForCacheGroupsAndNames(cacheGroputToNames);
- assertEquals("Unexpected number of lines in output.", 20, outputStr.split("\n").length);
+ assertContains(
+ log,
+ outputStr,
+ "Indexes rebuild was started for these caches:" + U.nl() + caches
+ );
}
/**
@@ -657,4 +761,26 @@ public class GridCommandHandlerIndexForceRebuildTest extends GridCommandHandlerA
return idxRebuildFuts.get(cacheId);
}
+
+ /**
+ * Force rebuilds indices for chosen caches, and waits until rebuild process is complete.
+ *
+ * @param cacheNames Cache names need indices to rebuild.
+ * @param grid Ignite node.
+ * @throws Exception If failed.
+ */
+ private void forceRebuildIndices(Iterable<String> cacheNames, IgniteEx grid) throws Exception {
+ String cacheNamesArg = String.join(",", cacheNames);
+
+ assertEquals(
+ EXIT_CODE_OK,
+ execute(
+ "--cache", "indexes_force_rebuild",
+ "--node-id", grid.localNode().id().toString(),
+ "--cache-names", cacheNamesArg
+ )
+ );
+
+ waitForIndexesRebuild(grid, getTestTimeout(), Collections.emptyList());
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 5acc048..8dc4a75 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -4310,7 +4310,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param <V> type of values.
* @return Cache instance for given name.
*/
- public <K, V> IgniteInternalCache<K, V> cache(String name) {
+ public <K, V> @Nullable IgniteInternalCache<K, V> cache(String name) {
assert name != null;
if (log.isDebugEnabled())
@@ -4465,6 +4465,7 @@ public class GridCacheProcessor extends GridProcessorAdapter {
* @param <K> type of keys.
* @param <V> type of values.
* @return Cache instance for given name.
+ * @throws IllegalArgumentException If cache not exists.
*/
public <K, V> IgniteInternalCache<K, V> publicCache(String name) {
assert name != null;
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java
index 23ad2d2..1b55ed3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/cache/index/IndexForceRebuildTask.java
@@ -17,22 +17,19 @@
package org.apache.ignite.internal.visor.cache.index;
+import java.util.Collection;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.visor.VisorJob;
import org.apache.ignite.internal.visor.VisorOneNodeTask;
-import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
@@ -67,112 +64,54 @@ public class IndexForceRebuildTask extends VisorOneNodeTask<IndexForceRebuildTas
@Override protected IndexForceRebuildTaskRes run(@Nullable IndexForceRebuildTaskArg arg)
throws IgniteException
{
- //Either cacheNames or cacheGrps must be specified
- assert (arg.cacheNames() == null) != (arg.cacheGrps() == null) :
- "Either cacheNames or cacheGroups must be specified";
-
- // Collect info about indexes being rebuilt.
- Set<IndexRebuildStatusInfoContainer> rebuildIdxCaches =
- ignite.context().cache().publicCaches()
- .stream()
- .filter(c -> !c.indexReadyFuture().isDone())
- .map(this::fromIgniteCache)
- .collect(Collectors.toSet());
-
- Set<String> rebuildIdxCachesNames = rebuildIdxCaches.stream()
- .map(IndexRebuildStatusInfoContainer::cacheName)
- .collect(Collectors.toSet());
-
- if (arg.cacheNames() != null)
- return rebuildByCacheNames(arg.cacheNames(), rebuildIdxCaches, rebuildIdxCachesNames);
- else if (arg.cacheGrps() != null)
- return rebuildByGroupNames(arg.cacheGrps(), rebuildIdxCaches, rebuildIdxCachesNames);
- else {
- assert false : "Neither cache names nor cache groups specified";
+ assert (arg.cacheNames() == null) ^ (arg.cacheGrps() == null) :
+ "Either cacheNames or cacheGroups must be specified.";
- return null;
- }
- }
+ Set<GridCacheContext> cachesToRebuild = new HashSet<>();
+ Set<String> notFound = new HashSet<>();
- /**
- * Triggers force rebuild of indexes in caches from {@code cacheNames}.
- *
- * @param cacheNames Set of cache names.
- * @param rebuildIdxCaches Set of infos about cached which have indexes being rebuilt at the moment.
- * @param rebuildIdxCachesNames Set of names of cached which have indexes being rebuilt at the moment.
- * @return {@code IndexForceRebuildTaskRes} object.
- */
- @NotNull private IndexForceRebuildTaskRes rebuildByCacheNames(
- Set<String> cacheNames,
- Set<IndexRebuildStatusInfoContainer> rebuildIdxCaches,
- Set<String> rebuildIdxCachesNames)
- {
final GridCacheProcessor cacheProcessor = ignite.context().cache();
- // Collect info about not found caches.
- Set<String> notFoundCaches = new HashSet<>(cacheNames);
- notFoundCaches.removeIf(name -> cacheProcessor.cache(name) != null);
-
- Set<GridCacheContext> cacheContexts =
- cacheProcessor.publicCaches()
- .stream()
- .filter(c -> !rebuildIdxCachesNames.contains(c.getName()))
- .filter(c -> cacheNames.contains(c.getName()))
- .map(IgniteCacheProxy::context)
- .collect(Collectors.toSet());
-
- // Collect info about started index rebuild.
- Set<IndexRebuildStatusInfoContainer> cachesWithStartedRebuild =
- cacheContexts.stream()
- .map(c -> new IndexRebuildStatusInfoContainer(c.config()))
- .collect(Collectors.toSet());
+ if (arg.cacheNames() != null) {
+ for (String cacheName : arg.cacheNames()) {
+ IgniteInternalCache cache = cacheProcessor.cache(cacheName);
- cacheProcessor.context().database().forceRebuildIndexes(cacheContexts);
+ if (cache != null)
+ cachesToRebuild.add(cache.context());
+ else
+ notFound.add(cacheName);
+ }
+ }
+ else {
+ for (String cacheGrpName : arg.cacheGrps()) {
+ CacheGroupContext grpCtx = cacheProcessor.cacheGroup(CU.cacheId(cacheGrpName));
+
+ if (grpCtx != null)
+ cachesToRebuild.addAll(grpCtx.caches());
+ else
+ notFound.add(cacheGrpName);
+ }
+ }
- return new IndexForceRebuildTaskRes(cachesWithStartedRebuild, rebuildIdxCaches, notFoundCaches);
- }
+ Collection<GridCacheContext> cachesCtxWithRebuildingInProgress =
+ ignite.context().cache().context().database().forceRebuildIndexes(cachesToRebuild);
- /**
- * Triggers force rebuild of indexes in all caches from {@code grpNames}.
- *
- * @param grpNames Set of cache groups names.
- * @param rebuildIdxCaches Set of infos about cached which have indexes being rebuilt at the moment.
- * @param rebuildIdxCachesNames Set of names of cached which have indexes being rebuilt at the moment.
- * @return {@code IndexForceRebuildTaskRes} object.
- */
- @NotNull private IndexForceRebuildTaskRes rebuildByGroupNames(
- Set<String> grpNames,
- Set<IndexRebuildStatusInfoContainer> rebuildIdxCaches,
- Set<String> rebuildIdxCachesNames)
- {
- final GridCacheProcessor cacheProcessor = ignite.context().cache();
-
- // Collect info about not found groups.
- Set<String> notFoundGroups = new HashSet<>(grpNames);
- notFoundGroups.removeIf(grpName -> cacheProcessor.cacheGroup(CU.cacheId(grpName)) != null);
-
- Set<GridCacheContext> cacheContexts =
- cacheProcessor.cacheGroups()
- .stream()
- .filter(grpContext -> grpNames.contains(grpContext.name()))
- .map(CacheGroupContext::caches)
- .flatMap(List::stream)
- .filter(c -> !rebuildIdxCachesNames.contains(c.name()))
+ Set<IndexRebuildStatusInfoContainer> cachesWithRebuildingInProgress =
+ cachesCtxWithRebuildingInProgress.stream()
+ .map(c -> new IndexRebuildStatusInfoContainer(c.config()))
.collect(Collectors.toSet());
Set<IndexRebuildStatusInfoContainer> cachesWithStartedRebuild =
- cacheContexts.stream()
+ cachesToRebuild.stream()
.map(c -> new IndexRebuildStatusInfoContainer(c.config()))
+ .filter(c -> !cachesWithRebuildingInProgress.contains(c))
.collect(Collectors.toSet());
- cacheProcessor.context().database().forceRebuildIndexes(cacheContexts);
-
- return new IndexForceRebuildTaskRes(cachesWithStartedRebuild, rebuildIdxCaches, notFoundGroups);
- }
-
- /** */
- private IndexRebuildStatusInfoContainer fromIgniteCache(IgniteCache c) {
- return new IndexRebuildStatusInfoContainer((CacheConfiguration)c.getConfiguration(CacheConfiguration.class));
+ return new IndexForceRebuildTaskRes(
+ cachesWithStartedRebuild,
+ cachesWithRebuildingInProgress,
+ notFound
+ );
}
}
}