You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2019/03/29 15:40:02 UTC
[ignite] branch master updated: IGNITE-11606 Fix index update after
index.bin manual removal - Fixes #6325.
This is an automated email from the ASF dual-hosted git repository.
agoncharuk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 9a9c817 IGNITE-11606 Fix index update after index.bin manual removal - Fixes #6325.
9a9c817 is described below
commit 9a9c817f8fb663e9bbfaab76d3d08058d30597d1
Author: EdShangGG <es...@gridgain.com>
AuthorDate: Fri Mar 29 18:23:55 2019 +0300
IGNITE-11606 Fix index update after index.bin manual removal - Fixes #6325.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
---
.../junits/IgniteCompatibilityAbstractTest.java | 2 +-
.../GridCacheDatabaseSharedManager.java | 17 ++
.../persistence/file/FilePageStoreManager.java | 6 +-
.../persistence/filename/PdsFolderSettings.java | 6 +
.../cache/verify/IdleVerifyResultV2.java | 5 +
.../processors/query/GridQueryIndexing.java | 7 +
.../processors/query/GridQueryProcessor.java | 36 ++-
.../verify/VisorValidateIndexesJobResult.java | 4 +
.../IgniteClientCacheInitializationFailTest.java | 5 +
.../testframework/junits/GridAbstractTest.java | 24 +-
.../junits/multijvm/IgniteProcessProxy.java | 32 ++-
.../IgniteHadoopFileSystemClientBasedOpenTest.java | 2 +-
.../processors/query/h2/IgniteH2Indexing.java | 7 +
.../processors/query/h2/opt/GridH2Table.java | 39 ++-
.../visor/verify/ValidateIndexesClosure.java | 2 +-
.../query/h2/GridIndexFullRebuildTest.java | 288 +++++++++++++++++++++
.../IgniteBinaryCacheQueryTestSuite.java | 2 +
17 files changed, 436 insertions(+), 48 deletions(-)
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java
index 826edba..d940664 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testframework/junits/IgniteCompatibilityAbstractTest.java
@@ -141,7 +141,7 @@ public abstract class IgniteCompatibilityAbstractTest extends GridCommonAbstract
final IgniteConfiguration cfg = getConfiguration(igniteInstanceName); // stub - won't be used at node startup
- IgniteProcessProxy ignite = new IgniteProcessProxy(cfg, log, locJvmInstance, true) {
+ IgniteProcessProxy ignite = new IgniteProcessProxy(cfg, log, locJvmInstance == null ? null: (x) -> locJvmInstance, true) {
@Override protected IgniteLogger logger(IgniteLogger log, Object ctgr) {
return ListenedGridTestLog4jLogger.createLogger(ctgr + "#" + ver.replaceAll("\\.", "_"));
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 7c7c07b..cece907 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -2232,6 +2232,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
int groupId = pageSnapshot.fullPageId().groupId();
int partId = partId(pageSnapshot.fullPageId().pageId());
+ if (skipRemovedIndexUpdates(groupId, partId))
+ break;
+
stripedApplyPage((pageMem) -> {
try {
applyPageSnapshot(pageMem, pageSnapshot);
@@ -2299,6 +2302,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
int groupId = pageDelta.groupId();
int partId = partId(pageDelta.pageId());
+ if (skipRemovedIndexUpdates(groupId, partId))
+ break;
+
stripedApplyPage((pageMem) -> {
try {
applyPageDelta(pageMem, pageDelta);
@@ -2518,6 +2524,14 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * @param grpId Group id.
+ * @param partId Partition id.
+ */
+ private boolean skipRemovedIndexUpdates(int grpId, int partId) {
+ return (partId == PageIdAllocator.INDEX_PARTITION) && !storeMgr.hasIndexStore(grpId);
+ }
+
+ /**
* Obtains PageMemory reference from cache descriptor instead of cache context.
*
* @param grpId Cache group id.
@@ -2689,6 +2703,9 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
stripedApply(() -> {
GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+ if (skipRemovedIndexUpdates(cacheCtx.groupId(), PageIdAllocator.INDEX_PARTITION))
+ cctx.kernalContext().query().markAsRebuildNeeded(cacheCtx);
+
try {
applyUpdate(cacheCtx, dataEntry);
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index 6c008d6..d20f5df 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -472,14 +472,14 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
/**
* Will preserve crc in buffer if keepCrc is true.
*
- * @param cacheId Cache ID.
+ * @param grpId Group ID.
* @param pageId Page ID.
* @param pageBuf Page buffer.
* @param keepCrc Keep CRC flag.
* @throws IgniteCheckedException If failed.
*/
- public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
- PageStore store = getStore(cacheId, PageIdUtils.partId(pageId));
+ public void read(int grpId, long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
+ PageStore store = getStore(grpId, PageIdUtils.partId(pageId));
try {
store.read(pageId, pageBuf, keepCrc);
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderSettings.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderSettings.java
index 20fb5ca..c47cbc9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderSettings.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/filename/PdsFolderSettings.java
@@ -20,6 +20,7 @@ package org.apache.ignite.internal.processors.cache.persistence.filename;
import java.io.File;
import java.io.Serializable;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -135,4 +136,9 @@ public class PdsFolderSettings {
@Nullable public File persistentStoreRootPath() {
return persistentStoreRootPath;
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PdsFolderSettings.class, this);
+ }
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
index 3ccfe00..3600f69 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/verify/IdleVerifyResultV2.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -50,15 +51,19 @@ public class IdleVerifyResultV2 extends VisorDataTransferObject {
private static final long serialVersionUID = 0L;
/** Counter conflicts. */
+ @GridToStringInclude
private Map<PartitionKeyV2, List<PartitionHashRecordV2>> cntrConflicts;
/** Hash conflicts. */
+ @GridToStringInclude
private Map<PartitionKeyV2, List<PartitionHashRecordV2>> hashConflicts;
/** Moving partitions. */
+ @GridToStringInclude
private Map<PartitionKeyV2, List<PartitionHashRecordV2>> movingPartitions;
/** Exceptions. */
+ @GridToStringInclude
private Map<ClusterNode, Exception> exceptions;
/**
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
index c36af42..74ab8e9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryIndexing.java
@@ -287,6 +287,13 @@ public interface GridQueryIndexing {
public IgniteInternalFuture<?> rebuildIndexesFromHash(GridCacheContext cctx);
/**
+ * Mark as rebuild needed for the given cache.
+ *
+ * @param cctx Cache context.
+ */
+ void markAsRebuildNeeded(GridCacheContext cctx);
+
+ /**
* Returns backup filter.
*
* @param topVer Topology version.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index dd0f2bc..d91c2aa 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -1834,22 +1834,48 @@ public class GridQueryProcessor extends GridProcessorAdapter {
}
/**
- * Rebuilds indexes for provided caches from corresponding hash indexes.
+ * Mark that for given cache index should/would be rebuilt.
*
* @param cctx Cache context.
- * @return Future that will be completed when rebuilding is finished.
*/
- public IgniteInternalFuture<?> rebuildIndexesFromHash(GridCacheContext cctx) {
+ public void markAsRebuildNeeded(GridCacheContext cctx) {
+ if (rebuildIsMeaningless(cctx))
+ return;
+
+ idx.markAsRebuildNeeded(cctx);
+ }
+
+ /**
+ * @param cctx Cache context.
+ * @return True if index rebuild is meaningless (index module is disabled, local node is not an affinity
+ * node fro this cache or queries are not enabled for the cache).
+ */
+ @SuppressWarnings("RedundantIfStatement")
+ private boolean rebuildIsMeaningless(GridCacheContext cctx) {
// Indexing module is disabled, nothing to rebuild.
if (idx == null)
- return null;
+ return true;
// No data on non-affinity nodes.
if (!cctx.affinityNode())
- return null;
+ return true;
// No indexes to rebuild when there are no QueryEntities.
if (!cctx.isQueryEnabled())
+ return true;
+
+ return false;
+ }
+
+ /**
+ * Rebuilds indexes for provided caches from corresponding hash indexes.
+ *
+ * @param cctx Cache context.
+ * @return Future that will be completed when rebuilding is finished.
+ */
+ public IgniteInternalFuture<?> rebuildIndexesFromHash(GridCacheContext cctx) {
+ // Indexing module is disabled, nothing to rebuild.
+ if (rebuildIsMeaningless(cctx))
return null;
// No need to rebuild if cache has no data.
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
index 6d7f765..99bccbd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/verify/VisorValidateIndexesJobResult.java
@@ -24,6 +24,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import org.apache.ignite.internal.processors.cache.verify.PartitionKey;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.internal.visor.VisorDataTransferObject;
@@ -37,12 +38,15 @@ public class VisorValidateIndexesJobResult extends VisorDataTransferObject {
private static final long serialVersionUID = 0L;
/** Results of indexes validation from node. */
+ @GridToStringInclude
private Map<PartitionKey, ValidateIndexesPartitionResult> partRes;
/** Results of reverse indexes validation from node. */
+ @GridToStringInclude
private Map<String, ValidateIndexesPartitionResult> idxRes;
/** Integrity check issues. */
+ @GridToStringInclude
private Collection<IndexIntegrityCheckIssue> integrityCheckFailures;
/**
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
index 9afb0e4..3f8009a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteClientCacheInitializationFailTest.java
@@ -381,6 +381,11 @@ public class IgniteClientCacheInitializationFailTest extends GridCommonAbstractT
}
/** {@inheritDoc} */
+ @Override public void markAsRebuildNeeded(GridCacheContext cctx) {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
@Override public IndexingQueryFilter backupFilter(AffinityTopologyVersion topVer, int[] parts) {
return null;
}
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 9244fd0..6b878c5 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -756,10 +756,10 @@ public abstract class GridAbstractTest extends JUnit3TestLegacySupport {
* @return First started grid.
* @throws Exception If failed.
*/
- protected final Ignite startGrids(int cnt) throws Exception {
+ protected final IgniteEx startGrids(int cnt) throws Exception {
assert cnt > 0;
- Ignite ignite = null;
+ IgniteEx ignite = null;
for (int i = 0; i < cnt; i++)
if (ignite == null)
@@ -998,7 +998,7 @@ public abstract class GridAbstractTest extends JUnit3TestLegacySupport {
}
}
else
- return startRemoteGrid(igniteInstanceName, null, ctx);
+ return startRemoteGrid(igniteInstanceName, cfg, ctx);
}
/**
@@ -1012,7 +1012,7 @@ public abstract class GridAbstractTest extends JUnit3TestLegacySupport {
*/
protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration cfg, GridSpringResourceContext ctx)
throws Exception {
- return startRemoteGrid(igniteInstanceName, cfg, ctx, grid(0), true);
+ return startRemoteGrid(igniteInstanceName, cfg, ctx,true);
}
/**
@@ -1083,20 +1083,24 @@ public abstract class GridAbstractTest extends JUnit3TestLegacySupport {
* @param igniteInstanceName Ignite instance name.
* @param cfg Ignite configuration.
* @param ctx Spring context.
- * @param locNode Local node.
* @param resetDiscovery Reset DiscoverySpi.
* @return Started grid.
* @throws Exception If failed.
*/
- protected Ignite startRemoteGrid(String igniteInstanceName, IgniteConfiguration cfg, GridSpringResourceContext ctx,
- IgniteEx locNode, boolean resetDiscovery)
- throws Exception {
+ protected Ignite startRemoteGrid(
+ String igniteInstanceName,
+ IgniteConfiguration cfg,
+ GridSpringResourceContext ctx,
+ boolean resetDiscovery
+ ) throws Exception {
if (ctx != null)
throw new UnsupportedOperationException("Starting of grid at another jvm by context doesn't supported.");
if (cfg == null)
cfg = optimize(getConfiguration(igniteInstanceName));
+ IgniteEx locNode = grid(0);
+
if (locNode != null) {
DiscoverySpi discoverySpi = locNode.configuration().getDiscoverySpi();
@@ -1117,7 +1121,7 @@ public abstract class GridAbstractTest extends JUnit3TestLegacySupport {
}
}
- return new IgniteProcessProxy(cfg, log, locNode, resetDiscovery, additionalRemoteJvmArgs());
+ return new IgniteProcessProxy(cfg, log, (x) -> grid(0), resetDiscovery, additionalRemoteJvmArgs());
}
/**
@@ -1179,7 +1183,7 @@ public abstract class GridAbstractTest extends JUnit3TestLegacySupport {
assert ignite != null : "Ignite returned null grid for name: " + igniteInstanceName;
- UUID id = ignite instanceof IgniteProcessProxy ? ignite.localNode().id() : ignite.context().localNodeId();
+ UUID id = ignite instanceof IgniteProcessProxy ? ((IgniteProcessProxy)ignite).getId() : ignite.context().localNodeId();
info(">>> Stopping grid [name=" + ignite.name() + ", id=" + id + ']');
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
index ccd4e9e..8d8546d 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/multijvm/IgniteProcessProxy.java
@@ -28,6 +28,7 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import javax.cache.CacheException;
import org.apache.ignite.DataRegionMetrics;
import org.apache.ignite.DataRegionMetricsAdapter;
@@ -119,7 +120,7 @@ public class IgniteProcessProxy implements IgniteEx {
private final transient IgniteConfiguration cfg;
/** Local JVM grid. */
- private final transient Ignite locJvmGrid;
+ private final transient Function<Void, Ignite> locJvmGrid;
/** Logger. */
private final transient IgniteLogger log;
@@ -135,7 +136,7 @@ public class IgniteProcessProxy implements IgniteEx {
*/
public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJvmGrid)
throws Exception {
- this(cfg, log, locJvmGrid, true);
+ this(cfg, log, (Function<Void, Ignite>)locJvmGrid, true);
}
/**
@@ -144,7 +145,7 @@ public class IgniteProcessProxy implements IgniteEx {
* @param locJvmGrid Local JVM grid.
* @throws Exception On error.
*/
- public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Ignite locJvmGrid, boolean discovery)
+ public IgniteProcessProxy(IgniteConfiguration cfg, IgniteLogger log, Function<Void, Ignite> locJvmGrid, boolean discovery)
throws Exception {
this(cfg, log, locJvmGrid, discovery, Collections.emptyList());
}
@@ -160,7 +161,7 @@ public class IgniteProcessProxy implements IgniteEx {
public IgniteProcessProxy(
IgniteConfiguration cfg,
IgniteLogger log,
- Ignite locJvmGrid,
+ Function<Void, Ignite> locJvmGrid,
boolean resetDiscovery,
List<String> additionalArgs
)
@@ -177,7 +178,8 @@ public class IgniteProcessProxy implements IgniteEx {
final CountDownLatch rmtNodeStartedLatch = new CountDownLatch(1);
if (locJvmGrid != null)
- locJvmGrid.events().localListen(new NodeStartedListener(id, rmtNodeStartedLatch), EventType.EVT_NODE_JOINED);
+ locJvmGrid.apply(null).events()
+ .localListen(new NodeStartedListener(id, rmtNodeStartedLatch), EventType.EVT_NODE_JOINED);
proc = GridJavaProcess.exec(
igniteNodeRunnerClassName(),
@@ -333,7 +335,7 @@ public class IgniteProcessProxy implements IgniteEx {
final CountDownLatch rmtNodeStoppedLatch = new CountDownLatch(1);
final UUID rmNodeId = proxy.getId();
- proxy.locJvmGrid.events().localListen(new IgnitePredicateX<Event>() {
+ proxy.localJvmGrid().events().localListen(new IgnitePredicateX<Event>() {
@Override public boolean applyx(Event e) {
if (((DiscoveryEvent)e).eventNode().id().equals(rmNodeId)) {
rmtNodeStoppedLatch.countDown();
@@ -358,6 +360,8 @@ public class IgniteProcessProxy implements IgniteEx {
throw t;
}
+ proxy.getProcess().kill();
+
gridProxies.remove(igniteInstanceName, proxy);
}
}
@@ -426,7 +430,7 @@ public class IgniteProcessProxy implements IgniteEx {
* @return Local JVM grid instance.
*/
public Ignite localJvmGrid() {
- return locJvmGrid;
+ return locJvmGrid.apply(null);
}
/**
@@ -685,7 +689,7 @@ public class IgniteProcessProxy implements IgniteEx {
/** {@inheritDoc} */
@Override public Collection<String> cacheNames() {
- return locJvmGrid.cacheNames();
+ return localJvmGrid().cacheNames();
}
/** {@inheritDoc} */
@@ -835,10 +839,10 @@ public class IgniteProcessProxy implements IgniteEx {
/** {@inheritDoc} */
@Override public void close() throws IgniteException {
- if (locJvmGrid != null) {
+ if (localJvmGrid() != null) {
final CountDownLatch rmtNodeStoppedLatch = new CountDownLatch(1);
- locJvmGrid.events().localListen(new IgnitePredicateX<Event>() {
+ localJvmGrid().events().localListen(new IgnitePredicateX<Event>() {
@Override public boolean applyx(Event e) {
if (((DiscoveryEvent)e).eventNode().id().equals(id)) {
rmtNodeStoppedLatch.countDown();
@@ -894,15 +898,17 @@ public class IgniteProcessProxy implements IgniteEx {
* @return {@link IgniteCompute} instance to communicate with remote node.
*/
public IgniteCompute remoteCompute() {
- if (locJvmGrid == null)
+ Ignite localJvmGrid = localJvmGrid();
+
+ if (localJvmGrid == null)
return null;
- ClusterGroup grp = locJvmGrid.cluster().forNodeId(id);
+ ClusterGroup grp = localJvmGrid.cluster().forNodeId(id);
if (grp.nodes().isEmpty())
throw new IllegalStateException("Could not found node with id=" + id + ".");
- return locJvmGrid.compute(grp);
+ return localJvmGrid.compute(grp);
}
/**
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
index 5e3221a..ef382c7 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/igfs/IgniteHadoopFileSystemClientBasedOpenTest.java
@@ -240,7 +240,7 @@ public class IgniteHadoopFileSystemClientBasedOpenTest extends GridCommonAbstrac
switch (nodeTypes[i]) {
case REMOTE:
startRemoteGrid(name, getConfiguration(name),
- null, null, false);
+ null, false);
break;
case LOCAL:
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index 4286e5a..cad0a4b 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -1795,6 +1795,13 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
/** {@inheritDoc} */
+ @Override public void markAsRebuildNeeded(GridCacheContext cctx) {
+ assert cctx.group().persistenceEnabled(): cctx;
+
+ markIndexRebuild(cctx.name(), true);
+ }
+
+ /** {@inheritDoc} */
@Override public IgniteInternalFuture<?> rebuildIndexesFromHash(GridCacheContext cctx) {
// No data in fresh in-memory cache.
if (!cctx.group().persistenceEnabled())
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index cb0adb9..b59dfb3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.LongAdder;
import java.util.concurrent.locks.Lock;
@@ -80,6 +81,16 @@ public class GridH2Table extends TableBase {
/** Exclusive lock constant. */
private static final long EXCLUSIVE_LOCK = -1;
+ /** 'rebuildFromHashInProgress' field updater */
+ private static final AtomicIntegerFieldUpdater<GridH2Table> rebuildFromHashInProgressFiledUpdater =
+ AtomicIntegerFieldUpdater.newUpdater(GridH2Table.class, "rebuildFromHashInProgress");
+
+ /** False representation */
+ private static final int FALSE = 0;
+
+ /** True representation */
+ private static final int TRUE = 1;
+
/** Cache context info. */
private final GridCacheContextInfo cacheInfo;
@@ -121,7 +132,7 @@ public class GridH2Table extends TableBase {
private final LongAdder size = new LongAdder();
/** */
- private volatile boolean rebuildFromHashInProgress;
+ private volatile int rebuildFromHashInProgress = FALSE;
/** Identifier. */
private final QueryTable identifier;
@@ -684,10 +695,10 @@ public class GridH2Table extends TableBase {
boolean replaced;
- if (prevRowAvailable)
+ if (prevRowAvailable && rebuildFromHashInProgress == FALSE)
replaced = pk().putx(row0);
else {
- prevRow0 = (H2CacheRow)pk().put(row0);
+ prevRow0 = pk().put(row0);
replaced = prevRow0 != null;
}
@@ -797,15 +808,15 @@ public class GridH2Table extends TableBase {
public void markRebuildFromHashInProgress(boolean value) {
assert !value || (idxs.size() >= 2 && index(1).getIndexType().isHash()) : "Table has no hash index.";
- rebuildFromHashInProgress = value;
+ if (rebuildFromHashInProgressFiledUpdater.compareAndSet(this, value? FALSE: TRUE, value ? TRUE: FALSE)) {
+ lock.writeLock().lock();
- lock.writeLock().lock();
-
- try {
- incrementModificationCounter();
- }
- finally {
- lock.writeLock().unlock();
+ try {
+ incrementModificationCounter();
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
}
}
@@ -813,7 +824,7 @@ public class GridH2Table extends TableBase {
*
*/
public boolean rebuildFromHashInProgress() {
- return rebuildFromHashInProgress;
+ return rebuildFromHashInProgress == TRUE;
}
/** {@inheritDoc} */
@@ -1009,7 +1020,7 @@ public class GridH2Table extends TableBase {
/** {@inheritDoc} */
@Override public Index getUniqueIndex() {
- if (rebuildFromHashInProgress)
+ if (rebuildFromHashInProgress == TRUE)
return index(1);
else
return index(2);
@@ -1017,7 +1028,7 @@ public class GridH2Table extends TableBase {
/** {@inheritDoc} */
@Override public ArrayList<Index> getIndexes() {
- if (!rebuildFromHashInProgress)
+ if (rebuildFromHashInProgress == FALSE)
return idxs;
ArrayList<Index> idxs = new ArrayList<>(2);
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
index b3c5fd3..10efba2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/visor/verify/ValidateIndexesClosure.java
@@ -557,7 +557,7 @@ public class ValidateIndexesClosure implements IgniteCallable<VisorValidateIndex
IndexValidationIssue is = new IndexValidationIssue(
o.toString(), cacheCtx.name(), idx.getName(), t);
- log.error("Failed to lookup key: " + is.toString());
+ log.error("Failed to lookup key: " + is.toString(), t);
enoughIssues |= partRes.reportIssue(is);
}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexFullRebuildTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexFullRebuildTest.java
new file mode 100644
index 0000000..2facb65
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/GridIndexFullRebuildTest.java
@@ -0,0 +1,288 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.query.h2;
+
+import com.google.common.collect.ImmutableSet;
+import java.io.File;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.ComputeTaskInternalFuture;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.visor.VisorTaskArgument;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesJobResult;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTask;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskArg;
+import org.apache.ignite.internal.visor.verify.VisorValidateIndexesTaskResult;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ *
+ */
+public class GridIndexFullRebuildTest extends GridCommonAbstractTest {
+ public static final String FIRST_CACHE = "cache1";
+ public static final String SECOND_CACHE = "cache2";
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration configuration = super.getConfiguration(igniteInstanceName);
+
+ configuration.setConsistentId(igniteInstanceName);
+
+ DataStorageConfiguration dsCfg = new DataStorageConfiguration();
+
+ dsCfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setMaxSize(256 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ );
+
+ dsCfg.setCheckpointFrequency(3_000);
+
+ configuration.setDataStorageConfiguration(dsCfg);
+
+ CacheConfiguration ccfgFirst = new CacheConfiguration();
+
+ LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+
+ fields.put("updateDate", "java.lang.Date");
+ fields.put("amount", "java.lang.Long");
+ fields.put("name", "java.lang.String");
+
+ Set<QueryIndex> indices = Collections.singleton(new QueryIndex("name", QueryIndexType.SORTED));
+
+ ccfgFirst.setName(FIRST_CACHE)
+ .setBackups(2)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setGroupName("group")
+ .setCacheMode(CacheMode.PARTITIONED)
+ .setQueryEntities(Collections.singletonList(
+ new QueryEntity(Long.class, Account.class)
+ .setFields(fields)
+ .setIndexes(indices)
+ ));
+
+ CacheConfiguration ccfgSecond = new CacheConfiguration(ccfgFirst).setName(SECOND_CACHE);
+
+ configuration.setCacheConfiguration(ccfgFirst, ccfgSecond);
+
+ return configuration;
+ }
+
+ /**
+ * We start several nodes, populate caches, then start replacing values.
+ * After that one node is killed, their index.bin files would be removed.
+ * Finally, we restart the node, index rebuild starting after recovery.
+ * And we checke indexes by "validate indexes" task.
+ */
+ @Test
+ public void test() throws Exception {
+
+ long start = System.currentTimeMillis();
+
+ IgniteEx grid1 = startGrids(4);
+
+ grid1.cluster().active(true);
+
+ final int accountCount = 2048;
+
+ try (IgniteDataStreamer streamer = grid1.dataStreamer(FIRST_CACHE)) {
+ for (long i = 0; i < accountCount; i++) {
+ streamer.addData(i, new Account(i));
+ }
+
+ streamer.flush();
+ }
+
+ try (IgniteDataStreamer streamer = grid1.dataStreamer(SECOND_CACHE)) {
+ for (long i = 0; i < accountCount; i++) {
+ streamer.addData(i, new Account(i));
+ }
+
+ streamer.flush();
+ }
+
+ AtomicBoolean stop = new AtomicBoolean();
+
+ IgniteCache<Object, Object> cache1 = grid1.cache(FIRST_CACHE);
+ IgniteCache<Object, Object> cache2 = grid1.cache(SECOND_CACHE);
+
+ new Thread(new Runnable() {
+ @Override public void run() {
+ long i = 0;
+
+ while (!stop.get()) {
+ try {
+ cache1.put(i, new Account(i));
+
+ cache2.put(i, new Account(i));
+
+ i++;
+ }
+ catch (Throwable e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }).start();
+
+ File workDirectory = U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false);
+
+ long diff = System.currentTimeMillis() - start;
+
+ U.sleep(7500 - (diff % 5000));
+
+ IgniteProcessProxy.kill(getTestIgniteInstanceName(3));
+
+ stop.set(true);
+
+ for (File grp : new File(workDirectory, U.maskForFileName(getTestIgniteInstanceName(3))).listFiles()) {
+ new File(grp, "index.bin").delete();
+ }
+
+ startGrid(3);
+
+ awaitPartitionMapExchange();
+
+ U.sleep(3_000);
+
+ ImmutableSet<UUID> nodes = ImmutableSet.of(((IgniteProcessProxy)grid(3)).getId(),
+ ((IgniteProcessProxy)grid(2)).getId());
+
+ VisorValidateIndexesTaskArg arg = new VisorValidateIndexesTaskArg(null,
+ null, 10000, 1);
+
+ VisorTaskArgument<VisorValidateIndexesTaskArg> argument = new VisorTaskArgument<>(nodes, arg, true);
+
+ ComputeTaskInternalFuture<VisorValidateIndexesTaskResult> execute = ((IgniteEx)grid1).context().task().execute(new VisorValidateIndexesTask(), argument);
+
+ VisorValidateIndexesTaskResult result = execute.get();
+
+ Map<UUID, VisorValidateIndexesJobResult> results = result.results();
+
+ boolean hasIssue = false;
+
+ for (VisorValidateIndexesJobResult jobResult : results.values()) {
+ System.err.println(jobResult);
+
+ hasIssue |= jobResult.hasIssues();
+ }
+
+ assertFalse(hasIssue);
+ }
+
+ /** */
+ private void cleanPersistenceFiles(String igName) throws Exception {
+ String ig1DbPath = Paths.get(DFLT_STORE_DIR, igName).toString();
+
+ File igDbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), ig1DbPath, false);
+
+ U.delete(igDbDir);
+
+ Files.createDirectory(igDbDir.toPath());
+
+ String ig1DbWalPath = Paths.get(DFLT_STORE_DIR, "wal", igName).toString();
+
+ U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), ig1DbWalPath, false));
+
+ ig1DbWalPath = Paths.get(DFLT_STORE_DIR, "wal", "archive", igName).toString();
+
+ U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(), ig1DbWalPath, false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean isMultiJvm() {
+ return true;
+ }
+
+ /** */
+ public class Account {
+ /** */
+ private Long id;
+ /** */
+ private String name;
+ /** */
+ private Long amount;
+ /** */
+ private Date updateDate;
+
+ /** */
+ public Account(Long id) {
+ this.id = id;
+
+ name = "Account" + id;
+ amount = id * 1000;
+ updateDate = new Date();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ Account account = (Account)o;
+ return Objects.equals(id, account.id);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(id);
+ }
+ }
+}
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
index 437b886..3a043a7 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteBinaryCacheQueryTestSuite.java
@@ -206,6 +206,7 @@ import org.apache.ignite.internal.processors.query.SqlQueryHistoryFromClientSelf
import org.apache.ignite.internal.processors.query.SqlQueryHistorySelfTest;
import org.apache.ignite.internal.processors.query.SqlSchemaSelfTest;
import org.apache.ignite.internal.processors.query.SqlSystemViewsSelfTest;
+import org.apache.ignite.internal.processors.query.h2.GridIndexFullRebuildTest;
import org.apache.ignite.internal.processors.query.h2.GridIndexRebuildSelfTest;
import org.apache.ignite.internal.processors.query.h2.H2ResultSetIteratorNullifyOnEndSelfTest;
import org.apache.ignite.internal.processors.query.h2.IgniteSqlBigIntegerKeyTest;
@@ -492,6 +493,7 @@ import org.junit.runners.Suite;
SqlSystemViewsSelfTest.class,
GridIndexRebuildSelfTest.class,
+ GridIndexFullRebuildTest.class,
SqlTransactionCommandsWithMvccDisabledSelfTest.class,