You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/03/19 06:27:23 UTC
[46/50] [abbrv] ignite git commit: IGNITE-7964 rmvId is stored to
MetaStorage metapage during operations - Fixes #3645.
IGNITE-7964 rmvId is stored to MetaStorage metapage during operations - Fixes #3645.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/892d8c73
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/892d8c73
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/892d8c73
Branch: refs/heads/ignite-zk
Commit: 892d8c739205c2b87598568042f0cffa3004d57b
Parents: f4ac81a
Author: Sergey Chugunov <se...@gmail.com>
Authored: Fri Mar 16 19:24:17 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Mar 16 19:24:17 2018 +0300
----------------------------------------------------------------------
.../GridCacheDatabaseSharedManager.java | 4 +-
.../persistence/metastorage/MetaStorage.java | 57 +++++--
.../metastorage/IgniteMetaStorageBasicTest.java | 147 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite.java | 4 +
4 files changed, 199 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 997f89a..45d9de5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -553,7 +553,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
try {
restoreMemory(status, true, storePageMem);
- metaStorage = new MetaStorage(cctx.wal(), regCfg, memMetrics, true);
+ metaStorage = new MetaStorage(cctx, regCfg, memMetrics, true);
metaStorage.init(this);
@@ -724,7 +724,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
cctx.pageStore().initializeForMetastorage();
- metaStorage = new MetaStorage(cctx.wal(), dataRegionMap.get(METASTORE_DATA_REGION_NAME),
+ metaStorage = new MetaStorage(cctx, dataRegionMap.get(METASTORE_DATA_REGION_NAME),
(DataRegionMetricsImpl)memMetricsMap.get(METASTORE_DATA_REGION_NAME));
WALPointer restore = restoreMemory(status);
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
index d57b939..d225659 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/MetaStorage.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
@@ -31,6 +32,7 @@ import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.MetastoreDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.IncompleteObject;
import org.apache.ignite.internal.processors.cache.persistence.DataRegion;
import org.apache.ignite.internal.processors.cache.persistence.DataRegionMetricsImpl;
@@ -79,6 +81,9 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
private final DataRegion dataRegion;
/** */
+ private final IgniteLogger log;
+
+ /** */
private MetastorageTree tree;
/** */
@@ -109,17 +114,18 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
private final Marshaller marshaller = new JdkMarshaller();
/** */
- public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
+ public MetaStorage(GridCacheSharedContext cctx, DataRegion dataRegion, DataRegionMetricsImpl regionMetrics,
boolean readOnly) {
- this.wal = wal;
+ wal = cctx.wal();
this.dataRegion = dataRegion;
this.regionMetrics = regionMetrics;
this.readOnly = readOnly;
+ log = cctx.logger(getClass());
}
/** */
- public MetaStorage(IgniteWriteAheadLogManager wal, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
- this(wal, memPlc, memMetrics, false);
+ public MetaStorage(GridCacheSharedContext cctx, DataRegion memPlc, DataRegionMetricsImpl memMetrics) {
+ this(cctx, memPlc, memMetrics, false);
}
/** */
@@ -309,6 +315,8 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
treeRoot = new RootPage(new FullPageId(io.getTreeRoot(pageAddr), METASTORAGE_CACHE_ID), false);
reuseListRoot = new RootPage(new FullPageId(io.getReuseListRoot(pageAddr), METASTORAGE_CACHE_ID), false);
+
+ rmvId.set(io.getGlobalRemoveId(pageAddr));
}
finally {
pageMem.readUnlock(METASTORAGE_CACHE_ID, partId, partMetaPage);
@@ -354,6 +362,8 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
treeRoot = io.getTreeRoot(pageAddr);
reuseListRoot = io.getReuseListRoot(pageAddr);
+ rmvId.set(io.getGlobalRemoveId(pageAddr));
+
assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
U.hexLong(treeRoot) + ", part=" + partId + ", METASTORAGE_CACHE_ID=" + METASTORAGE_CACHE_ID;
assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
@@ -384,21 +394,46 @@ public class MetaStorage implements DbCheckpointListener, ReadOnlyMetastorage, R
@Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
freeList.saveMetadata();
- MetastorageRowStore rowStore = tree.rowStore();
-
- saveStoreMetadata(rowStore, ctx);
+ saveStoreMetadata();
}
/**
- * @param rowStore Store to save metadata.
* @throws IgniteCheckedException If failed.
*/
- private void saveStoreMetadata(MetastorageRowStore rowStore, Context ctx) throws IgniteCheckedException {
- FreeListImpl freeList = (FreeListImpl)rowStore.freeList();
+ private void saveStoreMetadata() throws IgniteCheckedException {
+ PageMemoryEx pageMem = (PageMemoryEx) pageMemory();
- freeList.saveMetadata();
+ int partId = 0;
+
+ long partMetaId = pageMem.partitionMetaPageId(METASTORAGE_CACHE_ID, partId);
+ long partMetaPage = pageMem.acquirePage(METASTORAGE_CACHE_ID, partMetaId);
+
+ try {
+ long partMetaPageAddr = pageMem.writeLock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+
+ if (partMetaPageAddr == 0L) {
+ U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage + ']');
+
+ return;
+ }
+
+ boolean changed = false;
+
+ try {
+ PagePartitionMetaIO io = PageIO.getPageIO(partMetaPageAddr);
+
+ changed |= io.setGlobalRemoveId(partMetaPageAddr, rmvId.get());
+ }
+ finally {
+ pageMem.writeUnlock(METASTORAGE_CACHE_ID, partMetaId, partMetaPage, null, changed);
+ }
+ }
+ finally {
+ pageMem.releasePage(METASTORAGE_CACHE_ID, partMetaId, partMetaPage);
+ }
}
+ /** */
public void applyUpdate(String key, byte[] value) throws IgniteCheckedException {
if (readOnly) {
if (lastUpdates == null)
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
new file mode 100644
index 0000000..1837515
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/metastorage/IgniteMetaStorageBasicTest.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence.metastorage;
+
+import java.io.Serializable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ * Single place to add for basic MetaStorage tests.
+ */
+public class IgniteMetaStorageBasicTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ DataStorageConfiguration storageCfg = new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(
+ new DataRegionConfiguration()
+ .setMaxSize(100 * 1024 * 1024)
+ .setPersistenceEnabled(true)
+ )
+ .setWalMode(WALMode.LOG_ONLY);
+
+ cfg.setDataStorageConfiguration(storageCfg);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ cleanPersistenceDir();
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /**
+ * Verifies that MetaStorage after massive amounts of keys stored and updated keys restores its state successfully
+ * after restart.
+ *
+ * See <a hfer="https://issues.apache.org/jira/browse/IGNITE-7964" target="_top">IGNITE-7964</a> for more context
+ * about this test.
+ *
+ * @throws Exception If failed.
+ */
+ public void testMetaStorageMassivePutUpdateRestart() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ ig.cluster().active(true);
+
+ final byte KEYS_CNT = 100;
+ final String KEY_PREFIX = "test.key.";
+ final String NEW_VAL_PREFIX = "new.val.";
+ final String UPDATED_VAL_PREFIX = "updated.val.";
+
+ loadKeys(ig, KEYS_CNT, KEY_PREFIX, NEW_VAL_PREFIX, UPDATED_VAL_PREFIX);
+
+ stopGrid(0);
+
+ ig = startGrid(0);
+
+ ig.cluster().active(true);
+
+ verifyKeys(ig, KEYS_CNT, KEY_PREFIX, UPDATED_VAL_PREFIX);
+ }
+
+ /** */
+ private void loadKeys(IgniteEx ig,
+ byte keysCnt,
+ String keyPrefix,
+ String newValPrefix,
+ String updatedValPrefix
+ ) throws IgniteCheckedException {
+ IgniteCacheDatabaseSharedManager db = ig.context().cache().context().database();
+
+ MetaStorage metaStorage = db.metaStorage();
+
+ db.checkpointReadLock();
+ try {
+ for (byte i = 0; i < keysCnt; i++)
+ metaStorage.write(keyPrefix + i, newValPrefix + i);
+
+ for (byte i = 0; i < keysCnt; i++)
+ metaStorage.write(keyPrefix + i, updatedValPrefix + i);
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ /** */
+ private void verifyKeys(IgniteEx ig,
+ byte keysCnt,
+ String keyPrefix,
+ String valPrefix
+ ) throws IgniteCheckedException {
+ MetaStorage metaStorage = ig.context().cache().context().database().metaStorage();
+
+ for (byte i = 0; i < keysCnt; i++) {
+ Serializable val = metaStorage.read(keyPrefix + i);
+
+ Assert.assertEquals(valPrefix + i, val);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/892d8c73/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index a36f35b..a9668e7 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsDataR
import org.apache.ignite.internal.processors.cache.persistence.db.file.DefaultPageSizeBackwardsCompatibilityTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCheckpointSimulationWithRealCpDisabledTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsPageReplacementTest;
+import org.apache.ignite.internal.processors.cache.persistence.metastorage.IgniteMetaStorageBasicTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.FillFactorMetricTest;
@@ -114,5 +115,8 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsDataRegionMetricsTest.class);
suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
+
+ //MetaStorage
+ suite.addTestSuite(IgniteMetaStorageBasicTest.class);
}
}