You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ir...@apache.org on 2020/01/17 15:16:04 UTC
[ignite] branch master updated (f0b88f6 -> e31b530)
This is an automated email from the ASF dual-hosted git repository.
irakov pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git.
from f0b88f6 IGNITE-12535: Pass custom CipherSuites to JDBC thin client. (#7252)
new 1fbc284 IGNITE-12101 Fixed NullPointerException when IgniteQueue.removeAll is called. - Fixes #7266.
new e31b530 IGNITE-12531 Cluster is unable to change BLT on 2.8 if storage was initially created on 2.7 or less - Fixes #7265.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../IgnitePKIndexesMigrationToUnwrapPkTest.java | 1 -
.../persistence/MetaStorageCompatibilityTest.java | 348 +++++++++++++++++++++
.../IgniteCompatibilityBasicTestSuite.java | 6 +-
.../cache/persistence/metastorage/MetaStorage.java | 46 ++-
.../datastructures/GridCacheQueueAdapter.java | 4 +-
.../metastorage/IgniteMetaStorageBasicTest.java | 4 +-
.../GridCacheReplicatedQueueRemoveSelfTest.java | 144 +++++++++
.../IgniteCacheDataStructuresSelfTestSuite.java | 2 +
8 files changed, 534 insertions(+), 21 deletions(-)
create mode 100644 modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java
create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/datastructures/GridCacheReplicatedQueueRemoveSelfTest.java
[ignite] 01/02: IGNITE-12101 Fixed NullPointerException when
IgniteQueue.removeAll is called. - Fixes #7266.
Posted by ir...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 1fbc284fd0c89e6aaedf76bd2df3581cf85a1811
Author: Slava Koptilin <sl...@gmail.com>
AuthorDate: Fri Jan 17 16:46:56 2020 +0300
IGNITE-12101 Fixed NullPointerException when IgniteQueue.removeAll is called. - Fixes #7266.
Signed-off-by: Ivan Rakov <ir...@apache.org>
---
.../datastructures/GridCacheQueueAdapter.java | 4 +-
.../GridCacheReplicatedQueueRemoveSelfTest.java | 144 +++++++++++++++++++++
.../IgniteCacheDataStructuresSelfTestSuite.java | 2 +
3 files changed, 148 insertions(+), 2 deletions(-)
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java
index daf7b04..e7bde03 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/datastructures/GridCacheQueueAdapter.java
@@ -1045,13 +1045,13 @@ public abstract class GridCacheQueueAdapter<T> extends AbstractCollection<T> imp
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeGridUuid(out, id);
- out.writeLong(idx);
+ out.writeObject(idx);
}
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
id = U.readGridUuid(in);
- idx = in.readLong();
+ idx = (Long)in.readObject();
}
}
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastructures/GridCacheReplicatedQueueRemoveSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastructures/GridCacheReplicatedQueueRemoveSelfTest.java
new file mode 100644
index 0000000..1541510
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastructures/GridCacheReplicatedQueueRemoveSelfTest.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.datastructures;
+
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteQueue;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.configuration.CollectionConfiguration;
+import org.apache.ignite.internal.processors.cache.CacheInvokeEntry;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl;
+import org.apache.ignite.internal.processors.cache.datastructures.IgniteCollectionAbstractTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
+import org.apache.ignite.internal.util.io.GridByteArrayInputStream;
+import org.apache.ignite.internal.util.io.GridByteArrayOutputStream;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.junit.Test;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.REPLICATED;
+
+/**
+ * Tests for Ignite Queue remove method.
+ */
+public class GridCacheReplicatedQueueRemoveSelfTest extends IgniteCollectionAbstractTest {
+ /** */
+ public static final int CACHE_SIZE = 1_000;
+
+ /** */
+ public static final int THREADS_CNT = 8;
+
+ /** {@inheritDoc} */
+ @Override protected int gridCount() {
+ return 2;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheMode collectionCacheMode() {
+ return REPLICATED;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheAtomicityMode collectionCacheAtomicityMode() {
+ return TRANSACTIONAL;
+ }
+
+ /**
+ * This a unit test of Ignite queue's RemoveProcessor process() method.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testQueueRemovalProcessor() throws Exception {
+ GridCacheContext cctx = grid(0).context().cache().cache("ignite-sys-cache").context();
+
+ IgniteUuid id = IgniteUuid.randomUuid();
+
+ CacheInvokeEntry entry = new CacheInvokeEntry<>(null, null, null, false,
+ new GridDhtCacheEntry(cctx, null,
+ new KeyCacheObjectImpl(1, BigInteger.valueOf(1).toByteArray(), 1)));
+
+ entry.setValue(new GridCacheQueueHeader(
+ id,
+ 2147483647,
+ false,
+ 0L,
+ 10000L,
+ Collections.singleton(1L)));
+
+ GridCacheQueueAdapter.RemoveProcessor rp = new GridCacheQueueAdapter.RemoveProcessor(id, 1L);
+
+ rp.process(entry);
+
+ GridCacheQueueAdapter.RemoveProcessor externalRP = new GridCacheQueueAdapter.RemoveProcessor();
+
+ GridByteArrayOutputStream output = new GridByteArrayOutputStream();
+
+ rp.writeExternal(new ObjectOutputStream(output));
+
+ externalRP.readExternal(new ObjectInputStream(new GridByteArrayInputStream(output.toByteArray())));
+
+ assertEquals(id, GridTestUtils.getFieldValue(externalRP, "id"));
+
+ // idx should be null, cause entry was already removed, see GridCacheQueueAdapter.RemoveProcessor.code
+ // for more details.
+ assertNull(GridTestUtils.getFieldValue(externalRP, "idx"));
+ }
+
+ /**
+ * Removes buckets of data from the queue in many threads at the same time.
+ * @throws Exception If failed.
+ */
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testQueueRemoval() throws Exception {
+ IgniteQueue queue = grid(0).queue("SomeQueue", 0,
+ new CollectionConfiguration().setCollocated(true).setCacheMode(REPLICATED).setAtomicityMode(TRANSACTIONAL));
+
+ // Populate queue with some data.
+ for (int i = 0; i < CACHE_SIZE; i++)
+ queue.add(i);
+
+ CountDownLatch latch = new CountDownLatch(THREADS_CNT);
+
+ // Remove buckets of data from the queue in many threads at the same time.
+ GridTestUtils.runMultiThreaded(() -> {
+ ArrayList dataToRmv = new ArrayList();
+
+ for (int i = 0; i < CACHE_SIZE / 10; i++)
+ dataToRmv.add(ThreadLocalRandom.current().nextInt(CACHE_SIZE));
+
+ latch.countDown();
+ latch.await();
+
+ queue.removeAll(dataToRmv);
+
+ return null;
+ }, THREADS_CNT, "queue-test-worker");
+ }
+}
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
index a8b075e..42216ca 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheDataStructuresSelfTestSuite.java
@@ -91,6 +91,7 @@ import org.apache.ignite.internal.processors.cache.datastructures.replicated.Ign
import org.apache.ignite.internal.processors.cache.datastructures.replicated.IgniteReplicatedLockSelfTest;
import org.apache.ignite.internal.processors.cache.datastructures.replicated.IgniteReplicatedSemaphoreSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheAtomicReplicatedNodeRestartSelfTest;
+import org.apache.ignite.internal.processors.datastructures.GridCacheReplicatedQueueRemoveSelfTest;
import org.junit.runner.RunWith;
import org.junit.runners.Suite;
@@ -124,6 +125,7 @@ import org.junit.runners.Suite;
IgniteReplicatedSemaphoreSelfTest.class,
IgniteReplicatedLockSelfTest.class,
IgniteCacheAtomicReplicatedNodeRestartSelfTest.class,
+ GridCacheReplicatedQueueRemoveSelfTest.class,
GridCachePartitionedSequenceApiSelfTest.class,
GridCachePartitionedSequenceMultiNodeSelfTest.class,
[ignite] 02/02: IGNITE-12531 Cluster is unable to change BLT on 2.8
if storage was initially created on 2.7 or less - Fixes #7265.
Posted by ir...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
irakov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
commit e31b5300363e2c6a5f51374b8ba4bf6552fd2406
Author: Slava Koptilin <sl...@gmail.com>
AuthorDate: Fri Jan 17 18:14:07 2020 +0300
IGNITE-12531 Cluster is unable to change BLT on 2.8 if storage was initially created on 2.7 or less - Fixes #7265.
Signed-off-by: Ivan Rakov <ir...@apache.org>
---
.../IgnitePKIndexesMigrationToUnwrapPkTest.java | 1 -
.../persistence/MetaStorageCompatibilityTest.java | 348 +++++++++++++++++++++
.../IgniteCompatibilityBasicTestSuite.java | 6 +-
.../cache/persistence/metastorage/MetaStorage.java | 46 ++-
.../metastorage/IgniteMetaStorageBasicTest.java | 4 +-
5 files changed, 386 insertions(+), 19 deletions(-)
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java
index ed83ecd..3345d7b 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/IgnitePKIndexesMigrationToUnwrapPkTest.java
@@ -1,4 +1,3 @@
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java
new file mode 100644
index 0000000..0933808
--- /dev/null
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/persistence/MetaStorageCompatibilityTest.java
@@ -0,0 +1,348 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.ignite.compatibility.persistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.OpenOption;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Test;
+
+/**
+ * Tests migration of metastorage.
+ */
+public class MetaStorageCompatibilityTest extends IgnitePersistenceCompatibilityAbstractTest {
+ /** Consistent id. */
+ private static final String CONSISTENT_ID_1 = "node1";
+
+ /** Consistent id. */
+ private static final String CONSISTENT_ID_2 = "node2";
+
+ /** Ignite version. */
+ private static final String IGNITE_VERSION = "2.4.0";
+
+ /** Filename of index. */
+ private static final String INDEX_BIN_FILE = "index.bin";
+
+ /** Filename of partition. */
+ private static final String PART_FILE = "part-0.bin";
+
+ /**
+ * The zero partition hasn't full baseline information. The last changes are read from WAL.
+ *
+ * @throws Exception If failed.
+ */
+ @Test
+ public void testMigrationWithoutFullBaselineIntoPartition() throws Exception {
+ try {
+ U.delete(new File(U.defaultWorkDirectory()));
+
+ startGrid(1, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_1), new ActivateAndForceCheckpointClosure());
+
+ startGrid(2, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_2), new NewBaselineTopologyClosure());
+
+ stopAllGrids();
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) {
+ try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) {
+ // No-op.
+ }
+ }
+
+ assertFalse(metastorageFileExists(INDEX_BIN_FILE));
+ assertFalse(metastorageFileExists(PART_FILE));
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) {
+ try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) {
+ assertTrue(GridTestUtils.waitForCondition(() -> ig1.cluster().active(), 10_000));
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Tests that BLT can be changed and persisted after metastorage migration.
+ */
+ @Test
+ public void testMigrationToNewBaselineSetNewBaselineAfterMigration() throws Exception {
+ try {
+ U.delete(new File(U.defaultWorkDirectory()));
+
+ startGrid(1, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_1), new ActivateAndStopClosure());
+
+ stopAllGrids();
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) {
+ try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) {
+ ig0.cluster().setBaselineTopology(ig1.cluster().topologyVersion());
+ }
+ }
+
+ assertFalse(metastorageFileExists(INDEX_BIN_FILE));
+ assertFalse(metastorageFileExists(PART_FILE));
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) {
+ try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) {
+ assertTrue(GridTestUtils.waitForCondition(() -> ig1.cluster().active(), 10_000));
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ @Test
+ public void testMigrationWithExceptionDuringTheProcess() throws Exception {
+ try {
+ U.delete(new File(U.defaultWorkDirectory()));
+
+ startGrid(1, IGNITE_VERSION, new ConfigurationClosure(CONSISTENT_ID_1), new ActivateAndStopClosure());
+
+ stopAllGrids();
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1, PART_FILE))) {
+ ig0.getOrCreateCache("default-cache").put(1, 1); // trigger checkpoint on close()
+ }
+
+ assertTrue(metastorageFileExists(INDEX_BIN_FILE));
+ assertTrue(metastorageFileExists(PART_FILE));
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1, INDEX_BIN_FILE))) {
+ ig0.getOrCreateCache("default-cache").put(1, 1); // trigger checkpoint on close()
+ }
+
+ assertTrue(metastorageFileExists(INDEX_BIN_FILE));
+ assertFalse(metastorageFileExists(PART_FILE));
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) {
+ try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) {
+ ig0.cluster().setBaselineTopology(ig1.cluster().topologyVersion());
+ }
+ }
+
+ try (Ignite ig0 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_1))) {
+ try (Ignite ig1 = IgnitionEx.start(prepareConfig(getConfiguration(), CONSISTENT_ID_2))) {
+ assertTrue(GridTestUtils.waitForCondition(() -> ig1.cluster().active(), 10_000));
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Checks that file exists.
+ *
+ * @param fileName File name.
+ */
+ private boolean metastorageFileExists(String fileName) throws IgniteCheckedException {
+ return new File(U.defaultWorkDirectory() + "/db/" + U.maskForFileName(CONSISTENT_ID_1) + "/metastorage/" + fileName).exists();
+ }
+
+ /**
+ * Updates the given ignite configuration.
+ *
+ * @param cfg Ignite configuration to be updated.
+ * @param consistentId Consistent id.
+ * @return Updated configuration.
+ */
+ private static IgniteConfiguration prepareConfig(IgniteConfiguration cfg, @Nullable String consistentId) {
+ return prepareConfig(cfg, consistentId, null);
+ }
+
+ /**
+ * Updates the given ignite configuration.
+ *
+ * @param cfg Ignite configuration to be updated.
+ * @param consistentId Consistent id.
+ * @param failOnFileRmv Indicates that an exception should be trown when partition/index file is going to be removed.
+ * @return Updated configuration.
+ */
+ private static IgniteConfiguration prepareConfig(
+ IgniteConfiguration cfg,
+ @Nullable String consistentId,
+ @Nullable String failOnFileRmv
+ ) {
+ cfg.setLocalHost("127.0.0.1");
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+ disco.setIpFinder(GridCacheAbstractFullApiSelfTest.LOCAL_IP_FINDER);
+
+ cfg.setDiscoverySpi(disco);
+
+ cfg.setPeerClassLoadingEnabled(false);
+
+ DataStorageConfiguration memCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setInitialSize(10L * 1024 * 1024)
+ .setMaxSize(10L * 1024 * 1024))
+ .setPageSize(4096);
+
+ cfg.setDataStorageConfiguration(memCfg);
+
+ if (consistentId != null) {
+ cfg.setIgniteInstanceName(consistentId);
+ cfg.setConsistentId(consistentId);
+ }
+
+ if (failOnFileRmv != null)
+ cfg.getDataStorageConfiguration().setFileIOFactory(new FailingFileIOFactory(failOnFileRmv));
+
+ return cfg;
+ }
+
+ /** */
+ private static class ConfigurationClosure implements IgniteInClosure<IgniteConfiguration> {
+ /** Consistent id. */
+ private final String consistentId;
+
+ /**
+ * Creates a new instance of Configuration closure.
+ *
+ * @param consistentId Consistent id.
+ */
+ public ConfigurationClosure(String consistentId) {
+ this.consistentId = consistentId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void apply(IgniteConfiguration cfg) {
+ prepareConfig(cfg, consistentId, null);
+ }
+ }
+
+ /**
+ * Post-startup close that activate the grid and force checkpoint.
+ */
+ private static class ActivateAndForceCheckpointClosure implements IgniteInClosure<Ignite> {
+ /** {@inheritDoc} */
+ @Override public void apply(Ignite ignite) {
+ try {
+ ignite.active(true);
+
+ ((IgniteEx)ignite).context().cache().context().database()
+ .forceCheckpoint("force test checkpoint").finishFuture().get();
+
+ ((GridCacheDatabaseSharedManager)(((IgniteEx)ignite).context().cache().context().database()))
+ .enableCheckpoints(false);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+ }
+
+ /**
+ * Activates the cluster and stops it after that.
+ */
+ private static class ActivateAndStopClosure implements IgniteInClosure<Ignite> {
+ /** {@inheritDoc} */
+ @Override public void apply(Ignite ignite) {
+ ignite.active(true);
+
+ ignite.close();
+ }
+ }
+
+ /**
+ * Disables checkpointer and sets a new baseline topology.
+ */
+ private static class NewBaselineTopologyClosure implements IgniteInClosure<Ignite> {
+ /** {@inheritDoc} */
+ @Override public void apply(Ignite ignite) {
+ ((GridCacheDatabaseSharedManager)(((IgniteEx)ignite).context().cache().context().database()))
+ .enableCheckpoints(false);
+
+ long newTopVer = ignite.cluster().topologyVersion();
+
+ ignite.cluster().setBaselineTopology(newTopVer);
+ }
+ }
+
+ /**
+ * Create File I/O which fails after second attempt to write to File
+ */
+ private static class FailingFileIOFactory implements FileIOFactory {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Indicates that an exception should be trown when this file is going to be removed. */
+ private final String failOnFileRmv;
+
+ /** */
+ private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+ /**
+ * Creates a new instance of IO factory.
+ */
+ public FailingFileIOFactory() {
+ failOnFileRmv = null;
+ }
+
+ /**
+ * Creates a new instance of IO factory.
+ *
+ * @param failOnFileRmv Indicates that an exception should be trown when the given file is removed.
+ */
+ public FailingFileIOFactory(String failOnFileRmv) {
+ this.failOnFileRmv = failOnFileRmv;
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ final FileIO delegate = delegateFactory.create(file, modes);
+
+ return new FileIODecorator(delegate) {
+ @Override public void clear() throws IOException {
+ if (failOnFileRmv != null && failOnFileRmv.equals(file.getName()))
+ throw new IOException("Test remove fail!");
+
+ super.clear();
+ }
+ };
+ }
+ }
+}
diff --git a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
index 19ad466..e940ff4 100644
--- a/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
+++ b/modules/compatibility/src/test/java/org/apache/ignite/compatibility/testsuites/IgniteCompatibilityBasicTestSuite.java
@@ -18,6 +18,7 @@
package org.apache.ignite.compatibility.testsuites;
import org.apache.ignite.compatibility.persistence.FoldersReuseCompatibilityTest;
+import org.apache.ignite.compatibility.persistence.MetaStorageCompatibilityTest;
import org.apache.ignite.compatibility.persistence.MigratingToWalV2SerializerWithCompactionTest;
import org.apache.ignite.compatibility.persistence.PersistenceBasicCompatibilityTest;
import org.junit.runner.RunWith;
@@ -29,10 +30,9 @@ import org.junit.runners.Suite;
@RunWith(Suite.class)
@Suite.SuiteClasses({
PersistenceBasicCompatibilityTest.class,
-
FoldersReuseCompatibilityTest.class,
-
- MigratingToWalV2SerializerWithCompactionTest.class
+ MigratingToWalV2SerializerWithCompactionTest.class,
+ MetaStorageCompatibilityTest.class
})
public class IgniteCompatibilityBasicTestSuite {
}
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index 8db6d2c..f437126 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -44,12 +44,12 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.store.PageStore;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
import org.apache.ignite.internal.processors.cache.CacheDiagnosticManager;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
@@ -59,6 +59,7 @@ import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabase
import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.RootPage;
import org.apache.ignite.internal.processors.cache.persistence.StorageException;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
import org.apache.ignite.internal.processors.cache.persistence.partstorage.PartitionMetaStorageImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
@@ -177,14 +178,29 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage {
gcProcessor.setTmpStorage(null);
- // remove old partitions
- CacheGroupContext cgc = cctx.cache().cacheGroup(METASTORAGE_CACHE_ID);
+ db.addCheckpointListener(new DbCheckpointListener() {
+ /** {@inheritDoc} */
+ @Override public void onMarkCheckpointBegin(Context ctx) {
+ }
- if (cgc != null) {
- db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION);
+ /** {@inheritDoc} */
+ @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
+ assert cctx.pageStore() != null;
- db.schedulePartitionDestroy(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION);
- }
+ int partTag = ((PageMemoryEx)dataRegion.pageMemory()).invalidate(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION);
+ cctx.pageStore().onPartitionDestroyed(METASTORAGE_CACHE_ID, OLD_METASTORE_PARTITION, partTag);
+
+ int idxTag = ((PageMemoryEx)dataRegion.pageMemory()).invalidate(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION);
+ PageStore store = ((FilePageStoreManager)cctx.pageStore()).getStore(METASTORAGE_CACHE_ID, PageIdAllocator.INDEX_PARTITION);
+ store.truncate(idxTag);
+
+ db.removeCheckpointListener(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeCheckpointBegin(Context ctx) {
+ }
+ });
}
}
}
@@ -391,7 +407,7 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage {
while (cur.next()) {
MetastorageDataRow row = cur.get();
- res.add(new IgniteBiTuple<>(row.key(), row.value()));
+ res.add(new IgniteBiTuple<>(row.key(), marshaller.unmarshal(row.value(), getClass().getClassLoader())));
}
return res;
@@ -403,7 +419,15 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage {
byte[] data = marshaller.marshal(val);
- writeRaw(key, data);
+ final WALPointer ptr;
+
+ synchronized (this) {
+ ptr = wal.log(new MetastoreDataRecord(key, data));
+
+ writeRaw(key, data);
+ }
+
+ wal.flush(ptr, false);
}
/** {@inheritDoc} */
@@ -414,10 +438,6 @@ public class MetaStorage implements DbCheckpointListener, ReadWriteMetastorage {
/** {@inheritDoc} */
@Override public void writeRaw(String key, byte[] data) throws IgniteCheckedException {
if (!readOnly) {
- WALPointer ptr = wal.log(new MetastoreDataRecord(key, data));
-
- wal.flush(ptr, false);
-
synchronized (this) {
MetastorageDataRow oldRow = tree.findOne(new MetastorageDataRow(key, null));
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
index 730d8e9..4361c4b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -171,7 +171,7 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
for (Iterator<IgniteBiTuple<String, byte[]>> it = generateTestData(size, from).iterator(); it.hasNext(); ) {
IgniteBiTuple<String, byte[]> d = it.next();
- metaStorage.writeRaw(d.getKey(), d.getValue());
+ metaStorage.write(d.getKey(), d.getValue());
res.put(d.getKey(), d.getValue());
}
@@ -315,7 +315,7 @@ public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
try {
for (Map.Entry<String, byte[]> v : testData.entrySet())
- metaStorage.writeRaw(v.getKey(), v.getValue());
+ metaStorage.write(v.getKey(), v.getValue());
}
finally {
db.checkpointReadUnlock();