You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by dp...@apache.org on 2019/08/20 11:19:10 UTC
[ignite] branch ignite-2.7.6 updated: IGNITE-9562 Destroyed cache
that resurrected on an old offline node breaks PME - Fixes #6781.
This is an automated email from the ASF dual-hosted git repository.
dpavlov pushed a commit to branch ignite-2.7.6
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/ignite-2.7.6 by this push:
new a44d2a1 IGNITE-9562 Destroyed cache that resurrected on an old offline node breaks PME - Fixes #6781.
a44d2a1 is described below
commit a44d2a16af512e6ed256aececd876c512e4fe404
Author: EdShangGG <es...@gridgain.com>
AuthorDate: Tue Aug 20 14:18:50 2019 +0300
IGNITE-9562 Destroyed cache that resurrected on an old offline node breaks PME - Fixes #6781.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
---
.../cache/CacheJoinNodeDiscoveryData.java | 7 +
.../processors/cache/ClusterCachesInfo.java | 49 ++++
.../processors/cache/GridCacheProcessor.java | 23 +-
.../distributed/CacheBaselineTopologyTest.java | 6 +-
.../persistence/IgnitePdsDestroyCacheTest.java | 12 +-
.../GridMarshallerMappingConsistencyTest.java | 2 +-
.../testframework/junits/GridAbstractTest.java | 4 +-
.../db/IgniteCacheGroupsWithRestartsTest.java | 251 +++++++++++++++++++++
8 files changed, 344 insertions(+), 10 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
index bb0b59b..c7a59a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheJoinNodeDiscoveryData.java
@@ -162,6 +162,13 @@ public class CacheJoinNodeDiscoveryData implements Serializable {
}
/**
+ * @return Long which bits represent some flags.
+ */
+ public long getFlags() {
+ return flags;
+ }
+
+ /**
* @param ois ObjectInputStream.
*/
private void readObject(ObjectInputStream ois)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
index 6ab4e67..9d61eb6 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/ClusterCachesInfo.java
@@ -32,6 +32,7 @@ import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.stream.Collectors;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.cache.CacheExistsException;
@@ -939,6 +940,7 @@ class ClusterCachesInfo {
/**
* @return Information about started caches.
+ * @param cfgSplitter Cache configuration splitter.
*/
private CacheNodeCommonDiscoveryData collectCommonDiscoveryData() {
Map<Integer, CacheGroupData> cacheGrps = new HashMap<>();
@@ -1481,6 +1483,53 @@ class ClusterCachesInfo {
}
/**
+ * @param data Joining node data.
+ * @return Message with error or null if everything was OK.
+ */
+ public String validateJoiningNodeData(DiscoveryDataBag.JoiningNodeDiscoveryData data) {
+ if (data.hasJoiningNodeData()) {
+ Serializable joiningNodeData = data.joiningNodeData();
+
+ if (joiningNodeData instanceof CacheJoinNodeDiscoveryData) {
+ CacheJoinNodeDiscoveryData joinData = (CacheJoinNodeDiscoveryData)joiningNodeData;
+
+ Set<String> problemCaches = null;
+
+ for (CacheJoinNodeDiscoveryData.CacheInfo cacheInfo : joinData.caches().values()) {
+ CacheConfiguration<?, ?> cfg = cacheInfo.cacheData().config();
+
+ if (!registeredCaches.containsKey(cfg.getName())) {
+ String conflictErr = checkCacheConflict(cfg);
+
+ if (conflictErr != null) {
+ U.warn(log, "Ignore cache received from joining node. " + conflictErr);
+
+ continue;
+ }
+
+ long flags = cacheInfo.getFlags();
+
+ if (flags == 1L) {
+ if (problemCaches == null)
+ problemCaches = new HashSet<>();
+
+ problemCaches.add(cfg.getName());
+ }
+ }
+ }
+
+ if (!F.isEmpty(problemCaches))
+ return problemCaches.stream().collect(Collectors.joining(", ",
+ "Joining node has caches with data which are not presented on cluster, " +
+ "it could mean that they were already destroyed, to add the node to cluster - " +
+ "remove directories with the caches[", "]"));
+ }
+ }
+
+ return null;
+ }
+
+ /**
* @param clientData Discovery data.
* @param clientNodeId Client node ID.
*/
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index ec88a93..8adeb0b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -839,7 +839,7 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor
if (cacheType != CacheType.USER && cfg.getDataRegionName() == null)
cfg.setDataRegionName(sharedCtx.database().systemDateRegionName());
- addStoredCache(caches, cacheData, cacheName, cacheType, true);
+ addStoredCache(caches, cacheData, cacheName, cacheType, false, true);
}
}
@@ -852,14 +852,21 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor
* @param cacheType Cache type.
* @param isStaticalyConfigured Statically configured flag.
*/
- private void addStoredCache(Map<String, CacheInfo> caches, StoredCacheData cacheData, String cacheName,
- CacheType cacheType, boolean isStaticalyConfigured) {
+ private void addStoredCache(
+ Map<String, CacheInfo> caches,
+ StoredCacheData cacheData,
+ String cacheName,
+ CacheType cacheType,
+ boolean persistedBefore,
+ boolean isStaticalyConfigured
+ ) {
if (!cacheType.userCache())
stopSeq.addLast(cacheName);
else
stopSeq.addFirst(cacheName);
- caches.put(cacheName, new CacheInfo(cacheData, cacheType, cacheData.sql(), 0, isStaticalyConfigured));
+ caches.put(cacheName, new CacheJoinNodeDiscoveryData.CacheInfo(cacheData, cacheType, cacheData.sql(),
+ persistedBefore ? 1 : 0, isStaticalyConfigured));
}
/**
@@ -893,7 +900,7 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor
//Ignore stored caches if it already added by static config(static config has higher priority).
if (!caches.containsKey(cacheName))
- addStoredCache(caches, storedCacheData, cacheName, cacheType(cacheName), false);
+ addStoredCache(caches, storedCacheData, cacheName, cacheType(cacheName), true, false);
else {
CacheConfiguration cfg = caches.get(cacheName).cacheData().config();
CacheConfiguration cfgFromStore = storedCacheData.config();
@@ -2740,6 +2747,12 @@ public class GridCacheProcessor extends GridProcessorAdapter implements Metastor
if(!cachesInfo.isMergeConfigSupports(node))
return null;
+ String validationRes = cachesInfo.validateJoiningNodeData(discoData);
+
+ if (validationRes != null)
+ return new IgniteNodeValidationResult(node.id(), validationRes, validationRes);
+
+
if (discoData.hasJoiningNodeData() && discoData.joiningNodeData() instanceof CacheJoinNodeDiscoveryData) {
CacheJoinNodeDiscoveryData nodeData = (CacheJoinNodeDiscoveryData)discoData.joiningNodeData();
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
index 053ed82..7c41eeb 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/CacheBaselineTopologyTest.java
@@ -798,7 +798,11 @@ public class CacheBaselineTopologyTest extends GridCommonAbstractTest {
stopAllGrids();
- startGrids(5);
+ startGrid(1); //TODO https://issues.apache.org/jira/browse/IGNITE-8717 (replace with startGrids(5); //after)
+ startGrid(0);
+ startGrid(2);
+ startGrid(3);
+ startGrid(4);
GridTestUtils.waitForCondition(() -> grid(0).cluster().active(), getTestTimeout());
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
index 99d6f01..fe67f79 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDestroyCacheTest.java
@@ -18,6 +18,8 @@
package org.apache.ignite.internal.processors.cache.persistence;
import org.apache.ignite.Ignite;
+import org.junit.Ignore;
+import org.junit.Test;
/**
* Test correct clean up cache configuration data after destroying cache.
@@ -58,7 +60,11 @@ public class IgnitePdsDestroyCacheTest extends IgnitePdsDestroyCacheAbstractTest
*
* @throws Exception If failed.
*/
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-8717")
+ @Test
public void testDestroyCachesAbruptly() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8717");
+
Ignite ignite = startGrids(NODES);
ignite.cluster().active(true);
@@ -67,13 +73,17 @@ public class IgnitePdsDestroyCacheTest extends IgnitePdsDestroyCacheAbstractTest
checkDestroyCachesAbruptly(ignite);
}
-
+
/**
* Test destroy group caches abruptly with checkpoints.
*
* @throws Exception If failed.
*/
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-8717")
+ @Test
public void testDestroyGroupCachesAbruptly() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8717");
+
Ignite ignite = startGrids(NODES);
ignite.cluster().active(true);
diff --git a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
index ba39e36..07addd5 100644
--- a/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/marshaller/GridMarshallerMappingConsistencyTest.java
@@ -156,8 +156,8 @@ public class GridMarshallerMappingConsistencyTest extends GridCommonAbstractTest
stopAllGrids();
- Ignite g2 = startGrid(2);
startGrid(1);
+ Ignite g2 = startGrid(2);
assertTrue("Failed to wait for automatic grid activation",
GridTestUtils.waitForCondition(() -> g2.cluster().active(), getTestTimeout()));
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
index 13387de..a86f106 100755
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/GridAbstractTest.java
@@ -683,10 +683,10 @@ public abstract class GridAbstractTest extends TestCase {
* @return First started grid.
* @throws Exception If failed.
*/
- protected final Ignite startGrids(int cnt) throws Exception {
+ protected final IgniteEx startGrids(int cnt) throws Exception {
assert cnt > 0;
- Ignite ignite = null;
+ IgniteEx ignite = null;
for (int i = 0; i < cnt; i++)
if (ignite == null)
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java
new file mode 100644
index 0000000..08c5c4d
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteCacheGroupsWithRestartsTest.java
@@ -0,0 +1,251 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ConnectorConfiguration;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR;
+
+/**
+ * Testing corner cases in cache group functionality: -stopping cache in shared group and immediate node leaving;
+ * -starting cache in shared group with the same name as destroyed one; -etc.
+ */
+@SuppressWarnings({"unchecked", "ThrowableNotThrown"})
+public class IgniteCacheGroupsWithRestartsTest extends GridCommonAbstractTest {
+ /**
+ *
+ */
+ private volatile boolean startExtraStaticCache;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration configuration = super.getConfiguration(gridName);
+
+ configuration.setConsistentId(gridName);
+
+ configuration.setConnectorConfiguration(new ConnectorConfiguration());
+
+ DataStorageConfiguration cfg = new DataStorageConfiguration();
+
+ cfg.setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(256 * 1024 * 1024));
+
+ configuration.setDataStorageConfiguration(cfg);
+
+ if (startExtraStaticCache)
+ configuration.setCacheConfiguration(getCacheConfiguration(3));
+
+ return configuration;
+ }
+
+ /**
+ * @param i Cache index number.
+ * @return Cache configuration with the given number in name.
+ */
+ private CacheConfiguration<Object, Object> getCacheConfiguration(int i) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+
+ fields.put("updateDate", "java.lang.Date");
+ fields.put("amount", "java.lang.Long");
+ fields.put("name", "java.lang.String");
+
+ Set<QueryIndex> indices = Collections.singleton(new QueryIndex("name", QueryIndexType.SORTED));
+
+ ccfg.setName(getCacheName(i))
+ .setGroupName("group")
+ .setQueryEntities(Collections.singletonList(
+ new QueryEntity(Long.class, Account.class)
+ .setFields(fields)
+ .setIndexes(indices)
+ ))
+ .setAffinity(new RendezvousAffinityFunction(false, 64));
+
+ return ccfg;
+ }
+
+ /**
+ * @param i Index.
+ * @return Generated cache name for index.
+ */
+ private String getCacheName(int i) {
+ return "cache-" + i;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ afterTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Ignore("https://issues.apache.org/jira/browse/IGNITE-8717")
+ @Test
+ public void testNodeRestartRightAfterCacheStop() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-8717");
+
+ IgniteEx ex = startGrids(3);
+
+ prepareCachesAndData(ex);
+
+ ex.destroyCache(getCacheName(0));
+
+ assertNull(ex.cachex(getCacheName(0)));
+
+ stopGrid(2, true);
+
+ startGrid(2);
+
+ assertNull(ex.cachex(getCacheName(0)));
+
+ IgniteCache<Object, Object> cache = ex.createCache(getCacheConfiguration(0));
+
+ awaitPartitionMapExchange();
+
+ assertEquals(0, cache.size());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testNodeRestartBetweenCacheStop() throws Exception {
+ IgniteEx ex = startGrids(3);
+
+ prepareCachesAndData(ex);
+
+ stopGrid(2, true);
+
+ ex.destroyCache(getCacheName(0));
+
+ assertNull(ex.cachex(getCacheName(0)));
+
+ try {
+ startGrid(2);
+
+ fail();
+ }
+ catch (Exception e) {
+ List<Throwable> list = X.getThrowableList(e);
+
+ assertTrue(list.stream().
+ anyMatch(x -> x.getMessage().
+ contains("Joining node has caches with data which are not presented on cluster")));
+ }
+
+ removeCacheDir(getTestIgniteInstanceName(2), "cacheGroup-group");
+
+ IgniteEx node2 = startGrid(2);
+
+ assertEquals(3, node2.cluster().nodes().size());
+ }
+
+ /**
+ * @param instanceName Instance name.
+ * @param cacheGroup Cache group.
+ */
+ private void removeCacheDir(String instanceName, String cacheGroup) throws IgniteCheckedException {
+ String dn2DirName = instanceName.replace(".", "_");
+
+ U.delete(U.resolveWorkDirectory(U.defaultWorkDirectory(),
+ DFLT_STORE_DIR + "/" + dn2DirName + "/" + cacheGroup, true));
+ }
+
+ /**
+ * @param ignite Ignite instance.
+ */
+ private void prepareCachesAndData(IgniteEx ignite) {
+ ignite.cluster().active(true);
+
+ for (int j = 0; j < 3; j++) {
+ for (int i = 0; i < 64 * 10; i++) {
+ IgniteCache<Object, Object> cache = ignite.getOrCreateCache(getCacheConfiguration(j));
+
+ byte[] val = new byte[ThreadLocalRandom.current().nextInt(8148)];
+
+ Arrays.fill(val, (byte)i);
+
+ cache.put((long)i, new Account(i));
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ static class Account {
+ /**
+ *
+ */
+ private final int val;
+
+ /**
+ * @param val Value.
+ */
+ public Account(int val) {
+ this.val = val;
+ }
+
+ /**
+ * @return Value.
+ */
+ public int value() {
+ return val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Account.class, this);
+ }
+ }
+}