You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by ag...@apache.org on 2017/06/11 20:03:27 UTC
[01/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Repository: ignite
Updated Branches:
refs/heads/ignite-5267-merge-ea bdbba0ee9 -> 6bf5ce46c
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
deleted file mode 100644
index d0933fe..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.reconnect;
-
-import com.google.common.collect.Sets;
-import java.util.Collections;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import org.apache.ignite.internal.IgniteEx;
-
-/**
- *
- */
-public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstractStandByClientReconnectTest {
-
- public void testActiveClientReconnectToActiveCluster() throws Exception {
- CountDownLatch activateLatch = new CountDownLatch(1);
-
- startNodes(activateLatch);
-
- info(">>>> star grid");
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- info(">>>> activate grid");
-
- client.active(true);
-
- checkDescriptors(ig1, staticCacheNames);
- checkDescriptors(ig2, staticCacheNames);
- checkDescriptors(client, staticCacheNames);
-
- checkStaticCaches();
-
- info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- info(">>>> stop servers");
-
- stopGrid(node1);
- stopGrid(node2);
-
- disconnectedLatch.await();
-
- ig1 = startGrid(getConfiguration(node1));
- ig2 = startGrid(getConfiguration(node2));
-
- info(">>>> activate new servers");
-
- ig1.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
-
- activateLatch.countDown();
-
- info(">>>> reconnect client");
-
- reconnectedLatch.await();
-
- info(">>>> client reconnected");
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkAllCaches();
- }
-
- public void testActiveClientReconnectToInActiveCluster() throws Exception {
- startNodes(null);
-
- info(">>>> star grid");
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- info(">>>> activate grid");
-
- client.active(true);
-
- checkStaticCaches();
-
- checkDescriptors(ig1, staticCacheNames);
- checkDescriptors(ig2, staticCacheNames);
- checkDescriptors(client, staticCacheNames);
-
- info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- info(">>>> stop servers");
-
- stopGrid(node1);
- stopGrid(node2);
-
- assertTrue(client.active());
-
- System.out.println("Await disconnected");
-
- disconnectedLatch.await();
-
- ig1 = startGrid(getConfiguration("node1"));
- ig2 = startGrid(getConfiguration("node2"));
-
- info(">>>> reconnect client");
-
- reconnectedLatch.await();
-
- info(">>>> client reconnected");
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- info(">>>> activate new servers");
-
- client.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkAllCaches();
- }
-
- public void testInActiveClientReconnectToActiveCluster() throws Exception {
- CountDownLatch activateLatch = new CountDownLatch(1);
-
- startNodes(activateLatch);
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- stopGrid(node1);
- stopGrid(node2);
-
- disconnectedLatch.await();
-
- ig1 = startGrid(getConfiguration(node1));
- ig2 = startGrid(getConfiguration(node2));
-
- ig1.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
-
- checkDescriptors(ig1, Collections.<String>emptySet());
- checkDescriptors(ig2, Collections.<String>emptySet());
-
- activateLatch.countDown();
-
- reconnectedLatch.await();
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkOnlySystemCaches();
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- Set<String> exp2 = Sets.newHashSet(ccfgDynamicName, ccfgDynamicWithFilterName);
-
- checkDescriptors(ig1, exp2);
- checkDescriptors(ig2, exp2);
- checkDescriptors(client, exp2);
- }
-
- public void testInActiveClientReconnectToInActiveCluster() throws Exception {
- startNodes(null);
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- stopGrid(node1);
- stopGrid(node2);
-
- assertTrue(!client.active());
-
- disconnectedLatch.await();
-
- ig1 = startGrid(getConfiguration(node1));
- ig2 = startGrid(getConfiguration(node2));
-
- reconnectedLatch.await();
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- client.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkOnlySystemCaches();
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- Set<String> exp2 = Sets.newHashSet(ccfgDynamicName, ccfgDynamicWithFilterName);
-
- checkDescriptors(ig1, exp2);
- checkDescriptors(ig2, exp2);
- checkDescriptors(client, exp2);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java b/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java
deleted file mode 100644
index 59b7f39..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.testsuites;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.cache.database.db.wal.IgnitePdsWalTlbTest;
-
-/**
- *
- */
-public class IgnitePdsOutOfMemoryTestSuite extends TestSuite {
- /**
- * @return Suite.
- * @throws Exception If failed.
- */
- public static TestSuite suite() throws Exception {
- TestSuite suite = new TestSuite("Ignite Persistent Store OOM Test Suite");
-
- suite.addTestSuite(IgnitePdsWalTlbTest.class);
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
deleted file mode 100644
index d0c55c9..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.testsuites;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.cache.database.IgnitePersistentStoreCacheGroupsTest;
-import org.apache.ignite.cache.database.IgnitePdsClientNearCachePutGetTest;
-import org.apache.ignite.cache.database.IgnitePdsDynamicCacheTest;
-import org.apache.ignite.cache.database.IgnitePdsSingleNodePutGetPersistenceTest;
-import org.apache.ignite.cache.database.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest;
-import org.apache.ignite.cache.database.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest;
-import org.apache.ignite.cache.database.db.IgnitePdsMultiNodePutGetRestartTest;
-import org.apache.ignite.cache.database.db.IgnitePdsPageEvictionTest;
-import org.apache.ignite.cache.database.db.file.IgnitePdsCacheIntegrationTest;
-import org.apache.ignite.cache.database.db.wal.IgniteWalDirectoriesConfigurationTest;
-import org.apache.ignite.cache.database.db.file.ignitePdsCheckpointSimulationTest;
-import org.apache.ignite.cache.database.db.file.IgnitePdsEvictionTest;
-import org.apache.ignite.cache.database.pagemem.BPlusTreeReuseListPageMemoryImplTest;
-import org.apache.ignite.cache.database.pagemem.BPlusTreePageMemoryImplTest;
-import org.apache.ignite.cache.database.pagemem.MetadataStoragePageMemoryImplTest;
-import org.apache.ignite.cache.database.pagemem.PageMemoryImplNoLoadTest;
-import org.apache.ignite.cache.database.pagemem.PageMemoryImplTest;
-import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
-import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
-import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
-import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest;
-import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
-import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeTinyPutGetTest;
-import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
-
-
-/**
- *
- */
-public class IgnitePdsTestSuite extends TestSuite {
- /**
- * @return Suite.
- * @throws Exception If failed.
- */
- public static TestSuite suite() throws Exception {
- TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite");
-
- // Basic PageMemory tests.
- suite.addTestSuite(PageMemoryImplNoLoadTest.class);
- suite.addTestSuite(MetadataStoragePageMemoryImplTest.class);
- suite.addTestSuite(IgnitePdsEvictionTest.class);
- suite.addTestSuite(PageMemoryImplTest.class);
-
- // Checkpointing smoke-test.
- suite.addTestSuite(IgnitePdsCacheIntegrationTest.class);
- suite.addTestSuite(ignitePdsCheckpointSimulationTest.class);
-
- // BTree tests with store page memory.
- suite.addTestSuite(BPlusTreePageMemoryImplTest.class);
- suite.addTestSuite(BPlusTreeReuseListPageMemoryImplTest.class);
-
- // Basic API tests.
- suite.addTestSuite(IgniteDbSingleNodePutGetTest.class);
- suite.addTestSuite(IgniteDbSingleNodeWithIndexingPutGetTest.class);
- suite.addTestSuite(IgniteDbMultiNodePutGetTest.class);
- suite.addTestSuite(IgniteDbMultiNodeWithIndexingPutGetTest.class);
- suite.addTestSuite(IgniteDbSingleNodeTinyPutGetTest.class);
- suite.addTestSuite(IgniteDbDynamicCacheSelfTest.class);
- suite.addTestSuite(IgniteDbClientNearCachePutGetTest.class);
-
- // Persistence-enabled.
- suite.addTestSuite(IgnitePdsMultiNodePutGetRestartTest.class);
- suite.addTestSuite(IgnitePdsSingleNodePutGetPersistenceTest.class);
- suite.addTestSuite(IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.class);
- suite.addTestSuite(IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.class);
- suite.addTestSuite(IgnitePdsPageEvictionTest.class);
- suite.addTestSuite(IgnitePdsDynamicCacheTest.class);
- suite.addTestSuite(IgniteWalDirectoriesConfigurationTest.class);
- suite.addTestSuite(IgnitePdsClientNearCachePutGetTest.class);
-
- suite.addTestSuite(IgnitePersistentStoreCacheGroupsTest.class);
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
deleted file mode 100644
index acfac63..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.testsuites;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.cache.database.IgnitePdsAtomicCacheRebalancingTest;
-import org.apache.ignite.cache.database.IgnitePdsTxCacheRebalancingTest;
-import org.apache.ignite.cache.database.IgnitePdsContinuousRestartTest;
-import org.apache.ignite.cache.database.IgnitePdsPageSizesTest;
-import org.apache.ignite.cache.database.IgnitePdsRecoveryAfterFileCorruptionTest;
-import org.apache.ignite.cache.database.IgnitePersistenceMetricsSelfTest;
-import org.apache.ignite.cache.database.IgnitePersistentStoreDataStructuresTest;
-import org.apache.ignite.cache.database.IgnitePersistentStoreSchemaLoadTest;
-import org.apache.ignite.cache.database.db.IgnitePdsPageEvictionDuringPartitionClearTest;
-import org.apache.ignite.cache.database.db.IgnitePdsTransactionsHangTest;
-import org.apache.ignite.cache.database.db.IgnitePdsWholeClusterRestartTest;
-import org.apache.ignite.cache.database.db.IgnitePdsRebalancingOnNotStableTopologyTest;
-import org.apache.ignite.cache.database.db.file.IgnitePdsNoActualWalHistoryTest;
-import org.apache.ignite.cache.database.db.wal.IgniteWalHistoryReservationsTest;
-import org.apache.ignite.cache.database.db.wal.IgniteWalRecoveryTest;
-import org.apache.ignite.cache.database.db.wal.WalRecoveryTxLogicalRecordsTest;
-import org.apache.ignite.cache.database.db.wal.crc.IgniteDataIntegrityTests;
-
-/**
- *
- */
-public class IgnitePdsTestSuite2 extends TestSuite {
- /**
- * @return Suite.
- * @throws Exception If failed.
- */
- public static TestSuite suite() throws Exception {
- TestSuite suite = new TestSuite("Ignite persistent Store Test Suite 2");
-
- // Integrity test.
- suite.addTestSuite(IgniteDataIntegrityTests.class);
- suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
- suite.addTestSuite(IgnitePdsPageSizesTest.class);
-
- // Metrics test.
- suite.addTestSuite(IgnitePersistenceMetricsSelfTest.class);
-
- // WAL recovery test.
- suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class);
- suite.addTestSuite(IgniteWalRecoveryTest.class);
-
- suite.addTestSuite(IgnitePdsTransactionsHangTest.class);
-
- suite.addTestSuite(IgnitePdsNoActualWalHistoryTest.class);
-
- suite.addTestSuite(IgnitePdsRebalancingOnNotStableTopologyTest.class);
-
- suite.addTestSuite(IgnitePdsWholeClusterRestartTest.class);
-
- suite.addTestSuite(IgnitePdsPageEvictionDuringPartitionClearTest.class);
-
- // Rebalancing test
- suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
- suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
- suite.addTestSuite(IgniteWalHistoryReservationsTest.class);
-
- suite.addTestSuite(IgnitePdsContinuousRestartTest.class);
-
- suite.addTestSuite(IgnitePersistentStoreSchemaLoadTest.class);
- suite.addTestSuite(IgnitePersistentStoreDataStructuresTest.class);
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/pds/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
deleted file mode 100644
index 1d0754b..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.testsuites;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.cache.database.standbycluster.IgniteChangeGlobalStateCacheTest;
-import org.apache.ignite.cache.database.standbycluster.IgniteChangeGlobalStateDataStreamerTest;
-import org.apache.ignite.cache.database.standbycluster.IgniteChangeGlobalStateDataStructureTest;
-import org.apache.ignite.cache.database.standbycluster.IgniteChangeGlobalStateFailOverTest;
-import org.apache.ignite.cache.database.standbycluster.IgniteChangeGlobalStateTest;
-
-/**
- *
- */
-public class IgniteStandByClusterSuite extends TestSuite {
- /**
- *
- */
- public static TestSuite suite() {
- TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suit");
-
- suite.addTestSuite(IgniteChangeGlobalStateTest.class);
- suite.addTestSuite(IgniteChangeGlobalStateCacheTest.class);
- suite.addTestSuite(IgniteChangeGlobalStateDataStructureTest.class);
- suite.addTestSuite(IgniteChangeGlobalStateDataStreamerTest.class);
- suite.addTestSuite(IgniteChangeGlobalStateFailOverTest.class);
-// suite.addTestSuite(IgniteChangeGlobalStateServiceTest.class);
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index 270ece4..6ec74b0 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -107,6 +107,7 @@
<yammer.metrics.core.version>2.2.0</yammer.metrics.core.version>
<zkclient.version>0.5</zkclient.version>
<zookeeper.version>3.4.6</zookeeper.version>
+ <javassist.version>3.20.0-GA</javassist.version>
<!-- OSGI Manifest generation default property values -->
<osgi.import.package>*</osgi.import.package>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f18aedc..7fa3eef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -93,7 +93,6 @@
<module>modules/kubernetes</module>
<module>modules/zeromq</module>
<module>modules/rocketmq</module>
- <module>modules/pds</module>
</modules>
<profiles>
[24/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java
new file mode 100644
index 0000000..e855859
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRemoveDuringRebalancingTest.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsRemoveDuringRebalancingTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setCacheConfiguration(
+ new CacheConfiguration()
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setBackups(1)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setRebalanceMode(CacheRebalanceMode.SYNC)
+ );
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+ dbCfg.setPageSize(1024);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+ memPlcCfg.setSwapFilePath("db");
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setDiscoverySpi(
+ new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ U.delete(new File(U.getIgniteHome(), "db"));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ G.stopAll(true);
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ U.delete(new File(U.getIgniteHome(), "db"));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRemovesDuringRebalancing() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(null)) {
+ streamer.allowOverwrite(true);
+
+ for (int i = 0; i < 100_000; i++)
+ streamer.addData(i, i);
+ }
+
+ final IgniteCache<Object, Object> cache = ig.cache(null);
+
+ IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ return startGrid(1);
+ }
+ });
+
+ for (int i = 0; i < 100_000; i++)
+ cache.remove(i);
+
+ fut.get();
+
+ IgniteEx another = grid(1);
+
+ IgniteCache<Object, Object> cache1 = another.cache(null);
+
+ for (int i = 0; i < 100_000; i++)
+ assertNull(cache1.localPeek(i));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java
new file mode 100644
index 0000000..1f861c7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsSingleNodePutGetPersistenceTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class IgnitePdsSingleNodePutGetPersistenceTest extends IgniteDbSingleNodePutGetTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java
new file mode 100644
index 0000000..c641ea4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsTxCacheRebalancingTest.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ *
+ */
+public class IgnitePdsTxCacheRebalancingTest extends IgnitePdsCacheRebalancingAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected CacheConfiguration cacheConfiguration(String cacheName) {
+ CacheConfiguration ccfg = new CacheConfiguration(cacheName);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setCacheMode(CacheMode.PARTITIONED);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setBackups(1);
+ ccfg.setRebalanceDelay(10_000);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ return ccfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ explicitTx = false;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTopologyChangesWithConstantLoadExplicitTx() throws Exception {
+ explicitTx = true;
+
+ testTopologyChangesWithConstantLoad();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceMetricsSelfTest.java
new file mode 100644
index 0000000..72e0b61
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistenceMetricsSelfTest.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.Serializable;
+import java.util.Objects;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.MemoryMetrics;
+import org.apache.ignite.PersistenceMetrics;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.PAX;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgnitePersistenceMetricsSelfTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final String GROUP1 = "grp1";
+
+ /** */
+ private boolean activeOnStart = true;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setConsistentId(gridName);
+
+ cfg.setActiveOnStart(activeOnStart);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+ memCfg.setPageSize(1024);
+
+ memCfg.setDefaultMemoryPolicyName("dflt-plc");
+
+ MemoryPolicyConfiguration memPlc = new MemoryPolicyConfiguration();
+ memPlc.setName("dflt-plc");
+ memPlc.setMaxSize(10 * 1024 * 1024);
+ memPlc.setMetricsEnabled(true);
+
+ memCfg.setMemoryPolicies(memPlc);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration()
+ .setMetricsEnabled(true).setWalMode(WALMode.LOG_ONLY));
+
+ cfg.setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(false));
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ cfg.setCacheConfiguration(cacheConfiguration(GROUP1, "cache", PARTITIONED, ATOMIC, 1));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+
+ super.afterTest();
+ }
+
+ /**
+ * @param grpName Cache group name.
+ * @param name Cache name.
+ * @param cacheMode Cache mode.
+ * @param atomicityMode Atomicity mode.
+ * @param backups Backups number.
+ * @return Cache configuration.
+ */
+ private CacheConfiguration cacheConfiguration(
+ String grpName,
+ String name,
+ CacheMode cacheMode,
+ CacheAtomicityMode atomicityMode,
+ int backups
+ ) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setName(name);
+ ccfg.setGroupName(grpName);
+ ccfg.setAtomicityMode(atomicityMode);
+ ccfg.setBackups(backups);
+ ccfg.setCacheMode(cacheMode);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ return ccfg;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPersistenceMetrics() throws Exception {
+ final IgniteEx ig = startGrid(0);
+
+ try {
+ IgniteCache<Object, Object> cache = ig.cache("cache");
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, new Person("first-" + i, "last-" + i));
+
+ {
+ MemoryMetrics memMetrics = ig.memoryMetrics("dflt-plc");
+
+ assertNotNull(memMetrics);
+ assertTrue(memMetrics.getDirtyPages() > 0);
+ }
+
+ ig.context().cache().context().database().waitForCheckpoint("test");
+
+ GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ PersistenceMetrics pMetrics = ig.persistentStoreMetrics();
+
+ assertNotNull(pMetrics);
+
+ return pMetrics.getLastCheckpointTotalPagesNumber() != 0 &&
+ pMetrics.getLastCheckpointDataPagesNumber() != 0;
+ }
+ }, 5_000);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ static class Person implements Serializable {
+ /** */
+ @GridToStringInclude
+ @QuerySqlField(index = true, groups = "full_name")
+ private String fName;
+
+ /** */
+ @GridToStringInclude
+ @QuerySqlField(index = true, groups = "full_name")
+ private String lName;
+
+ /**
+ * @param fName First name.
+ * @param lName Last name.
+ */
+ public Person(String fName, String lName) {
+ this.fName = fName;
+ this.lName = lName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Person.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ Person person = (Person)o;
+
+ return Objects.equals(fName, person.fName) &&
+ Objects.equals(lName, person.lName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(fName, lName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreCacheGroupsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreCacheGroupsTest.java
new file mode 100644
index 0000000..57cc8d0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreCacheGroupsTest.java
@@ -0,0 +1,508 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import javax.cache.Cache;
+import javax.cache.expiry.ExpiryPolicy;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicy;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheMode.PARTITIONED;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+
+/**
+ *
+ */
+public class IgnitePersistentStoreCacheGroupsTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final String GROUP1 = "grp1";
+
+ /** */
+ private static final String GROUP2 = "grp2";
+
+ /** */
+ private CacheConfiguration[] ccfgs;
+
+ /** */
+ private boolean activeOnStart = true;
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setConsistentId(gridName);
+
+ cfg.setActiveOnStart(activeOnStart);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+ memCfg.setPageSize(1024);
+ memCfg.setDefaultMemoryPolicySize(10 * 1024 * 1024);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration().setWalMode(WALMode.LOG_ONLY));
+
+ cfg.setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(false));
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ if (ccfgs != null) {
+ cfg.setCacheConfiguration(ccfgs);
+
+ ccfgs = null;
+ }
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ GridTestUtils.deleteDbFiles();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClusterRestartStaticCaches1() throws Exception {
+ clusterRestart(1, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClusterRestartStaticCaches2() throws Exception {
+ clusterRestart(3, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClusterRestartDynamicCaches1() throws Exception {
+ clusterRestart(1, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testClusterRestartDynamicCaches2() throws Exception {
+ clusterRestart(3, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public void testClusterRestartCachesWithH2Indexes() throws Exception {
+ CacheConfiguration[] ccfgs1 = new CacheConfiguration[5];
+
+ // Several caches with the same indexed type (and index names).
+ ccfgs1[0] = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1).
+ setIndexedTypes(Integer.class, Person.class);
+ ccfgs1[1] = cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL, 1).
+ setIndexedTypes(Integer.class, Person.class);
+ ccfgs1[2] = cacheConfiguration(GROUP2, "c3", PARTITIONED, ATOMIC, 1).
+ setIndexedTypes(Integer.class, Person.class);
+ ccfgs1[3] = cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 1).
+ setIndexedTypes(Integer.class, Person.class);
+ ccfgs1[4] = cacheConfiguration(null, "c5", PARTITIONED, ATOMIC, 1).
+ setIndexedTypes(Integer.class, Person.class);
+
+ String[] caches = {"c1", "c2", "c3", "c4", "c5"};
+
+ startGrids(3);
+
+ Ignite node = ignite(0);
+
+ node.createCaches(Arrays.asList(ccfgs1));
+
+ putPersons(caches, node);
+
+ checkPersons(caches, node);
+ checkPersonsQuery(caches, node);
+
+ stopAllGrids();
+
+ startGrids(3);
+
+ awaitPartitionMapExchange();
+
+ node = ignite(0);
+
+ checkPersons(caches, node);
+ checkPersonsQuery(caches, node);
+
+ Random rnd = ThreadLocalRandom.current();
+
+ int idx = rnd.nextInt(caches.length);
+
+ String cacheName = caches[idx];
+ CacheConfiguration cacheCfg = ccfgs1[idx];
+
+ node.destroyCache(cacheName);
+
+ node.createCache(cacheCfg);
+
+ putPersons(new String[]{cacheName}, node);
+
+ checkPersons(caches, node);
+ checkPersonsQuery(caches, node);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void _testExpiryPolicy() throws Exception {
+ long ttl = 10000;
+
+ activeOnStart = false;
+
+ CacheConfiguration[] ccfgs1 = new CacheConfiguration[5];
+
+ ccfgs1[0] = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1);
+ ccfgs1[1] = cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL, 1);
+ ccfgs1[2] = cacheConfiguration(GROUP2, "c3", PARTITIONED, ATOMIC, 1);
+ ccfgs1[3] = cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 1);
+ ccfgs1[4] = cacheConfiguration(null, "c5", PARTITIONED, ATOMIC, 1);
+
+ String[] caches = {"c1", "c2", "c3", "c4", "c5"};
+
+ startGrids(3);
+
+ Ignite node = ignite(0);
+
+ node.active(true);
+
+ node.createCaches(Arrays.asList(ccfgs1));
+
+ ExpiryPolicy plc = new PlatformExpiryPolicy(ttl, -2, -2);
+
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName).withExpiryPolicy(plc);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, cacheName + i);
+ }
+
+ long deadline = System.currentTimeMillis() + (long)(ttl * 1.2);
+
+ stopAllGrids();
+
+ startGrids(3);
+
+ node = ignite(0);
+
+ node.active(true);
+
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ for (int i = 0; i < 10; i++)
+ assertEquals(cacheName + i, cache.get(i));
+
+ assertEquals(10, cache.size());
+ }
+
+ // Wait for expiration.
+ Thread.sleep(Math.max(deadline - System.currentTimeMillis(), 0));
+
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ assertEquals(0, cache.size());
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateDropCache() throws Exception {
+ ccfgs = new CacheConfiguration[]{cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1)
+ .setIndexedTypes(Integer.class, Person.class)};
+
+ Ignite ignite = startGrid();
+
+ ignite.cache("c1").destroy();
+
+ stopGrid();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateDropCache1() throws Exception {
+ CacheConfiguration ccfg1 = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1);
+
+ CacheConfiguration ccfg2 = cacheConfiguration(GROUP1, "c2", PARTITIONED, ATOMIC, 1);
+
+ Ignite ignite = startGrid();
+
+ ignite.createCaches(Arrays.asList(ccfg1, ccfg2));
+
+
+ ignite.cache("c1").destroy();
+
+ ignite.cache("c2").destroy();
+
+ ignite.createCache(ccfg1);
+ ignite.createCache(ccfg2);
+
+ stopGrid();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testCreateDropCache2() throws Exception {
+ CacheConfiguration ccfg1 = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1)
+ .setIndexedTypes(Integer.class, Person.class);
+
+ CacheConfiguration ccfg2 = cacheConfiguration(GROUP1, "c2", PARTITIONED, ATOMIC, 1)
+ .setIndexedTypes(Integer.class, Person.class);
+
+ Ignite ignite = startGrid();
+
+ ignite.createCaches(Arrays.asList(ccfg1, ccfg2));
+
+ ignite.cache("c1").destroy();
+
+ ignite.createCache(ccfg1);
+
+ stopGrid();
+ }
+
+ /**
+ * @param caches Cache names to put data into.
+ * @param node Ignite node.
+ */
+ private void putPersons(String[] caches, Ignite node) {
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, new Person("" + i, cacheName));
+ }
+ }
+
+ /**
+ * @param caches Cache names to invoke a query against to.
+ * @param node Ignite node.
+ */
+ private void checkPersons(String[] caches, Ignite node) {
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ for (int i = 0; i < 10; i++)
+ assertEquals(new Person("" + i, cacheName), cache.get(i));
+
+ assertEquals(10, cache.size());
+ }
+ }
+
+ /**
+ * @param caches Cache names to invoke a query against to.
+ * @param node Ignite node.
+ */
+ private void checkPersonsQuery(String[] caches, Ignite node) {
+ SqlQuery<Integer, Person> qry = new SqlQuery<>(
+ Person.class, "SELECT p.* FROM Person p WHERE p.lname=? ORDER BY p.fname");
+
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ List<Cache.Entry<Integer, Person>> persons = cache.query(qry.setArgs(cacheName)).getAll();
+
+ for (int i = 0; i < 10; i++)
+ assertEquals(new Person("" + i, cacheName), persons.get(i).getValue());
+
+ assertEquals(10, persons.size());
+ }
+ }
+
+ /**
+ * @param nodes Nodes number.
+ * @param staticCaches {@code True} if caches should be statically configured.
+ * @throws Exception If failed.
+ */
+ private void clusterRestart(int nodes, boolean staticCaches) throws Exception {
+ CacheConfiguration[] ccfgs = new CacheConfiguration[5];
+
+ ccfgs[0] = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1);
+ ccfgs[1] = cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL, 1);
+ ccfgs[2] = cacheConfiguration(GROUP2, "c3", PARTITIONED, ATOMIC, 1);
+ ccfgs[3] = cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 1);
+ ccfgs[4] = cacheConfiguration(null, "c5", PARTITIONED, ATOMIC, 1);
+
+ String[] caches = {"c1", "c2", "c3", "c4", "c5"};
+
+ for (int i = 0; i < nodes; i++) {
+ if (staticCaches)
+ this.ccfgs = ccfgs;
+
+ startGrid(i);
+ }
+
+ Ignite node = ignite(0);
+
+ if (!staticCaches)
+ node.createCaches(Arrays.asList(ccfgs));
+
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ for (int i = 0; i < 10; i++) {
+ cache.put(i, cacheName + i);
+
+ assertEquals(cacheName + i, cache.get(i));
+ }
+
+ assertEquals(10, cache.size());
+ }
+
+ stopAllGrids();
+
+ node = startGrids(nodes);
+
+ awaitPartitionMapExchange();
+
+ for (String cacheName : caches) {
+ IgniteCache<Object, Object> cache = node.cache(cacheName);
+
+ for (int i = 0; i < 10; i++)
+ assertEquals(cacheName + i, cache.get(i));
+
+ assertEquals(10, cache.size());
+ }
+ }
+
+ /**
+ * @param grpName Cache group name.
+ * @param name Cache name.
+ * @param cacheMode Cache mode.
+ * @param atomicityMode Atomicity mode.
+ * @param backups Backups number.
+ * @return Cache configuration.
+ */
+ private CacheConfiguration cacheConfiguration(
+ String grpName,
+ String name,
+ CacheMode cacheMode,
+ CacheAtomicityMode atomicityMode,
+ int backups
+ ) {
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setName(name);
+ ccfg.setGroupName(grpName);
+ ccfg.setAtomicityMode(atomicityMode);
+ ccfg.setBackups(backups);
+ ccfg.setCacheMode(cacheMode);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ return ccfg;
+ }
+
+ /**
+ *
+ */
+ static class Person implements Serializable {
+ /** */
+ @GridToStringInclude
+ @QuerySqlField(index = true, groups = "full_name")
+ String fName;
+
+ /** */
+ @GridToStringInclude
+ @QuerySqlField(index = true, groups = "full_name")
+ String lName;
+
+ /**
+ * @param fName First name.
+ * @param lName Last name.
+ */
+ public Person(String fName, String lName) {
+ this.fName = fName;
+ this.lName = lName;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Person.class, this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ Person person = (Person)o;
+ return Objects.equals(fName, person.fName) &&
+ Objects.equals(lName, person.lName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return Objects.hash(fName, lName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
new file mode 100644
index 0000000..6494903
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePersistentStoreDataStructuresTest.java
@@ -0,0 +1,196 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteAtomicLong;
+import org.apache.ignite.IgniteAtomicSequence;
+import org.apache.ignite.IgniteQueue;
+import org.apache.ignite.IgniteSet;
+import org.apache.ignite.configuration.CollectionConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(200 * 1024 * 1024);
+ memPlcCfg.setMaxSize(200 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration().setWalMode(WALMode.LOG_ONLY));
+
+ cfg.setActiveOnStart(false);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ GridTestUtils.deleteDbFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testQueue() throws Exception {
+ Ignite ignite = startGrids(4);
+
+ ignite.active(true);
+
+ IgniteQueue<Object> queue = ignite.queue("testQueue", 100, new CollectionConfiguration());
+
+ for (int i = 0; i < 100; i++)
+ queue.offer(i);
+
+ stopAllGrids();
+
+ ignite = startGrids(4);
+
+ ignite.active(true);
+
+ queue = ignite.queue("testQueue", 0, null);
+
+ for (int i = 0; i < 100; i++)
+ assertEquals(i, queue.poll());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testAtomic() throws Exception {
+ Ignite ignite = startGrids(4);
+
+ ignite.active(true);
+
+ IgniteAtomicLong atomicLong = ignite.atomicLong("testLong", 0, true);
+
+ for (int i = 0; i < 100; i++)
+ atomicLong.incrementAndGet();
+
+ stopAllGrids();
+
+ ignite = startGrids(4);
+
+ ignite.active(true);
+
+ atomicLong = ignite.atomicLong("testLong", 0, false);
+
+ for (int i = 100; i != 0; )
+ assertEquals(i--, atomicLong.getAndDecrement());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSequence() throws Exception {
+ Ignite ignite = startGrids(4);
+
+ ignite.active(true);
+
+ IgniteAtomicSequence sequence = ignite.atomicSequence("testSequence", 0, true);
+
+ int i = 0;
+
+ while (i < 1000) {
+ sequence.incrementAndGet();
+
+ i++;
+ }
+
+ stopAllGrids();
+
+ ignite = startGrids(4);
+
+ ignite.active(true);
+
+ sequence = ignite.atomicSequence("testSequence", 0, false);
+
+ assertTrue(sequence.incrementAndGet() > i);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSet() throws Exception {
+ Ignite ignite = startGrids(4);
+
+ ignite.active(true);
+
+ IgniteSet<Object> set = ignite.set("testSet", new CollectionConfiguration());
+
+ for (int i = 0; i < 100; i++)
+ set.add(i);
+
+ stopAllGrids();
+
+ ignite = startGrids(4);
+
+ ignite.active(true);
+
+ set = ignite.set("testSet", null);
+
+ assertFalse(set.add(99));
+
+ for (int i = 0; i < 100; i++)
+ assertTrue(set.contains(i));
+
+ assertEquals(100, set.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java
new file mode 100644
index 0000000..9a49b6c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicyInitializationTest.java
@@ -0,0 +1,313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Collection;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
+
+/**
+ *
+ */
+public class MemoryPolicyInitializationTest extends GridCommonAbstractTest {
+ /** */
+ private static final String CUSTOM_NON_DEFAULT_MEM_PLC_NAME = "custom_mem_plc";
+
+ /** */
+ private static final long USER_CUSTOM_MEM_PLC_SIZE = 10 * 1024 * 1024;
+
+ /** */
+ private static final long USER_DEFAULT_MEM_PLC_SIZE = 99 * 1024 * 1024;
+
+ /** */
+ private MemoryConfiguration memCfg;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ memCfg = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+ }
+
+ /**
+ * Verifies that expected memory policies are allocated when used doesn't provide any MemoryPolicyConfiguration.
+ */
+ public void testNoConfigProvided() throws Exception {
+ memCfg = null;
+
+ IgniteEx ignite = startGrid(0);
+
+ Collection<MemoryPolicy> allMemPlcs = ignite.context().cache().context().database().memoryPolicies();
+
+ assertTrue(allMemPlcs.size() == 2);
+
+ verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
+ }
+
+ /**
+ * Verifies that expected memory policies are allocated when used provides MemoryPolicyConfiguration
+ * with non-default custom MemoryPolicy.
+ */
+ public void testCustomConfigNoDefault() throws Exception {
+ prepareCustomNoDefaultConfig();
+
+ IgniteEx ignite = startGrid(0);
+
+ Collection<MemoryPolicy> allMemPlcs = ignite.context().cache().context().database().memoryPolicies();
+
+ assertTrue(allMemPlcs.size() == 3);
+
+ verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
+
+ assertTrue("Custom non-default memory policy is not presented",
+ isMemoryPolicyPresented(allMemPlcs, CUSTOM_NON_DEFAULT_MEM_PLC_NAME));
+ }
+
+ /**
+ * User is allowed to configure memory policy with 'default' name,
+ * in that case Ignite instance will use this user-defined memory policy as a default one.
+ */
+ public void testCustomConfigOverridesDefault() throws Exception {
+ prepareCustomConfigWithOverridingDefault();
+
+ IgniteEx ignite = startGrid(0);
+
+ IgniteCacheDatabaseSharedManager dbMgr = ignite.context().cache().context().database();
+
+ Collection<MemoryPolicy> allMemPlcs = dbMgr.memoryPolicies();
+
+ assertTrue(allMemPlcs.size() == 2);
+
+ verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
+
+ MemoryPolicy dfltMemPlc = U.field(dbMgr, "dfltMemPlc");
+
+ assertTrue(dfltMemPlc.config().getMaxSize() == USER_DEFAULT_MEM_PLC_SIZE);
+ }
+
+ /**
+ * User is allowed to define fully custom memory policy and make it default by setting its name to memory config.
+ *
+ * At the same time user still can create a memory policy with name 'default'
+ * which although won't be used as default.
+ */
+ public void testCustomConfigOverridesDefaultNameAndDeclaresDefault() throws Exception {
+ prepareCustomConfigWithOverriddenDefaultName();
+
+ IgniteEx ignite = startGrid(0);
+
+ IgniteCacheDatabaseSharedManager dbMgr = ignite.context().cache().context().database();
+
+ Collection<MemoryPolicy> allMemPlcs = dbMgr.memoryPolicies();
+
+ assertTrue(allMemPlcs.size() == 3);
+
+ verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
+
+ MemoryPolicy dfltMemPlc = U.field(dbMgr, "dfltMemPlc");
+
+ assertTrue(dfltMemPlc.config().getMaxSize() == USER_CUSTOM_MEM_PLC_SIZE);
+ }
+
+ /**
+ * Test for verification that caches with not specified memory policy name,
+ * with specified default memory policy name and specified custom memory policy name
+ * all started with correct memory policy.
+ */
+ public void testCachesOnOverriddenMemoryPolicy() throws Exception {
+ prepareCustomConfigWithOverridingDefaultAndCustom();
+
+ IgniteEx ignite = startGrid(0);
+
+ CacheConfiguration cache1Cfg = new CacheConfiguration()
+ .setName("cache1");
+
+ IgniteCache cache1 = ignite.createCache(cache1Cfg);
+
+ verifyCacheMemoryPolicy(cache1, DFLT_MEM_PLC_DEFAULT_NAME);
+
+ CacheConfiguration cache2Cfg = new CacheConfiguration()
+ .setName("cache2")
+ .setMemoryPolicyName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
+
+ IgniteCache cache2 = ignite.createCache(cache2Cfg);
+
+ verifyCacheMemoryPolicy(cache2, CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
+
+ CacheConfiguration cache3Cfg = new CacheConfiguration()
+ .setName("cache3")
+ .setMemoryPolicyName(DFLT_MEM_PLC_DEFAULT_NAME);
+
+ IgniteCache cache3 = ignite.createCache(cache3Cfg);
+
+ verifyCacheMemoryPolicy(cache3, DFLT_MEM_PLC_DEFAULT_NAME);
+ }
+
+ /**
+ * Test for verification that caches with not specified memory policy name,
+ * with specified default memory policy name and specified custom memory policy name
+ * all started with correct memory policy.
+ */
+ public void testCachesOnUserDefinedDefaultMemoryPolicy() throws Exception {
+ prepareCustomConfigWithOverriddenDefaultName();
+
+ IgniteEx ignite = startGrid(0);
+
+ CacheConfiguration cache1Cfg = new CacheConfiguration()
+ .setName("cache1");
+
+ IgniteCache cache1 = ignite.createCache(cache1Cfg);
+
+ verifyCacheMemoryPolicy(cache1, CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
+
+ CacheConfiguration cache2Cfg = new CacheConfiguration()
+ .setName("cache2")
+ .setMemoryPolicyName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
+
+ IgniteCache cache2 = ignite.createCache(cache2Cfg);
+
+ verifyCacheMemoryPolicy(cache2, CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
+
+ CacheConfiguration cache3Cfg = new CacheConfiguration()
+ .setName("cache3")
+ .setMemoryPolicyName(DFLT_MEM_PLC_DEFAULT_NAME);
+
+ IgniteCache cache3 = ignite.createCache(cache3Cfg);
+
+ verifyCacheMemoryPolicy(cache3, DFLT_MEM_PLC_DEFAULT_NAME);
+ }
+
+ /**
+ * @param cache Cache.
+ * @param plcName Policy name.
+ */
+ private void verifyCacheMemoryPolicy(IgniteCache cache, String plcName) {
+ GridCacheContext ctx = U.field(cache, "ctx");
+
+ assertEquals(plcName, ctx.memoryPolicy().config().getName());
+ }
+
+ /**
+ *
+ */
+ private void prepareCustomConfigWithOverriddenDefaultName() {
+ memCfg = new MemoryConfiguration();
+
+ memCfg.setDefaultMemoryPolicyName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
+
+ memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
+ .setName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME)
+ .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
+ .setMaxSize(USER_CUSTOM_MEM_PLC_SIZE),
+
+ new MemoryPolicyConfiguration()
+ .setName(DFLT_MEM_PLC_DEFAULT_NAME)
+ .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
+ .setMaxSize(USER_DEFAULT_MEM_PLC_SIZE)
+ );
+ }
+
+
+ /**
+ *
+ */
+ private void prepareCustomConfigWithOverridingDefault() {
+ memCfg = new MemoryConfiguration();
+
+ memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
+ .setName(DFLT_MEM_PLC_DEFAULT_NAME)
+ .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
+ .setMaxSize(USER_DEFAULT_MEM_PLC_SIZE)
+ );
+ }
+
+ /**
+ *
+ */
+ private void prepareCustomConfigWithOverridingDefaultAndCustom() {
+ memCfg = new MemoryConfiguration();
+
+ memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
+ .setName(DFLT_MEM_PLC_DEFAULT_NAME)
+ .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
+ .setMaxSize(USER_DEFAULT_MEM_PLC_SIZE),
+
+ new MemoryPolicyConfiguration()
+ .setName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME)
+ .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
+ .setMaxSize(USER_CUSTOM_MEM_PLC_SIZE)
+ );
+ }
+
+ /**
+ * @param allMemPlcs Collection of all memory policies.
+ */
+ private void verifyDefaultAndSystemMemoryPolicies(Collection<MemoryPolicy> allMemPlcs) {
+ assertTrue("Default memory policy is not presented",
+ isMemoryPolicyPresented(allMemPlcs, DFLT_MEM_PLC_DEFAULT_NAME));
+
+ assertTrue("System memory policy is not presented",
+ isMemoryPolicyPresented(allMemPlcs, IgniteCacheDatabaseSharedManager.SYSTEM_MEMORY_POLICY_NAME));
+ }
+
+ /**
+ *
+ */
+ private void prepareCustomNoDefaultConfig() {
+ memCfg = new MemoryConfiguration();
+
+ memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
+ .setName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME)
+ .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
+ .setMaxSize(USER_CUSTOM_MEM_PLC_SIZE)
+ );
+ }
+
+ /**
+ * @param memPlcs Collection of memory policies.
+ * @param nameToVerify Excepted name of memory policy.
+ */
+ private boolean isMemoryPolicyPresented(Collection<MemoryPolicy> memPlcs, String nameToVerify) {
+ for (MemoryPolicy memPlc : memPlcs) {
+ if (nameToVerify.equals(memPlc.config().getName()))
+ return true;
+ }
+
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsMultiNodePutGetRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsMultiNodePutGetRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsMultiNodePutGetRestartTest.java
new file mode 100644
index 0000000..e244333
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsMultiNodePutGetRestartTest.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.configuration.MemoryConfiguration;
+
+/**
+ *
+ */
+public class IgnitePdsMultiNodePutGetRestartTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int GRID_CNT = 3;
+
+ /** */
+ private static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+ dbCfg.setMemoryPolicies(memPlcCfg);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+ ccfg.setIndexedTypes(Integer.class, DbValue.class);
+
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration bCfg = new BinaryConfiguration();
+
+ bCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(bCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPutGetSimple() throws Exception {
+ String home = U.getIgniteHome();
+
+ File allocPath = new File(home, "work/db/" + UUID.randomUUID());
+
+ allocPath.mkdirs();
+
+ info(">>> Will use path: " + allocPath);
+
+ startGrids(GRID_CNT);
+
+ try {
+ IgniteEx ig = grid(0);
+
+ checkPutGetSql(ig, true);
+ }
+ finally {
+ stopAllGrids();
+ }
+
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+
+ startGrids(GRID_CNT);
+
+ try {
+ IgniteEx ig = grid(0);
+
+ checkPutGetSql(ig, false);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ig Ig.
+ * @param write Write.
+ */
+ private void checkPutGetSql(IgniteEx ig, boolean write) {
+ IgniteCache<Integer, DbValue> cache = ig.cache(CACHE_NAME);
+
+ if (write) {
+ try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(CACHE_NAME)) {
+ for (int i = 0; i < 10_000; i++)
+ streamer.addData(i, new DbValue(i, "value-" + i, i));
+ }
+ }
+
+ List<List<?>> res = cache.query(new SqlFieldsQuery("select ival from dbvalue where ival < ? order by ival asc")
+ .setArgs(10_000)).getAll();
+
+ assertEquals(10_000, res.size());
+
+ for (int i = 0; i < 10_000; i++) {
+ assertEquals(1, res.get(i).size());
+ assertEquals(i, res.get(i).get(0));
+ }
+
+ assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
+ assertEquals(5000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 10000"))
+ .getAll().size());
+
+ for (int i = 0; i < 10_000; i++)
+ assertEquals(new DbValue(i, "value-" + i, i), cache.get(i));
+ }
+
+ /**
+ *
+ */
+ private static class DbValue implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String sVal;
+
+ /** */
+ @QuerySqlField
+ private long lVal;
+
+ /**
+ * @param iVal Integer value.
+ * @param sVal String value.
+ * @param lVal Long value.
+ */
+ public DbValue(int iVal, String sVal, long lVal) {
+ this.iVal = iVal;
+ this.sVal = sVal;
+ this.lVal = lVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DbValue dbVal = (DbValue)o;
+
+ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DbValue.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionDuringPartitionClearTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionDuringPartitionClearTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionDuringPartitionClearTest.java
new file mode 100644
index 0000000..fb7113f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionDuringPartitionClearTest.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.concurrent.Callable;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsPageEvictionDuringPartitionClearTest extends GridCommonAbstractTest {
+ /** */
+ public static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
+ .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
+ .setAffinity(new RendezvousAffinityFunction(false, 128))
+ .setRebalanceMode(CacheRebalanceMode.SYNC)
+ .setBackups(1);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ // Intentionally set small page cache size.
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setInitialSize(70 * 1024 * 1024);
+ memPlcCfg.setMaxSize(70 * 1024 * 1024);
+
+ memPlcCfg.setName("dfltMemPlc");
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+
+ memCfg.setDefaultMemoryPolicyName(memPlcCfg.getName());
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 20 * 60 * 1000;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC, "true");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageEvictionOnNodeStart() throws Exception {
+ for (int r = 0; r < 3; r++) {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ startGrids(2);
+
+ try {
+ Ignite ig = ignite(0);
+
+ IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(CACHE_NAME);
+
+ for (int i = 0; i < 300_000; i++) {
+ streamer.addData(i, new TestValue(i));
+
+ if (i > 0 && i % 10_000 == 0)
+ info("Done: " + i);
+ }
+
+ streamer.flush();
+
+ IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgniteEx ig = startGrid(2);
+
+ info(">>>>>>>>>>>");
+ info(">>>>>>>>>>>");
+ info(">>>>>>>>>>>");
+
+ return ig;
+ }
+ });
+
+ for (int i = 500_000; i < 1_000_000; i++) {
+ streamer.addData(i, new TestValue(i));
+
+ if (i > 0 && i % 10_000 == 0) {
+ info("Done: " + i);
+
+ U.sleep(1000);
+ }
+ }
+
+ streamer.close();
+
+ fut.get();
+ }
+ finally {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestValue {
+ /** */
+ private int id;
+
+ /** */
+ private byte[] payload = new byte[512];
+
+ /**
+ * @param id ID.
+ */
+ private TestValue(int id) {
+ this.id = id;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionTest.java
new file mode 100644
index 0000000..e8bc701
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsPageEvictionTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.io.Serializable;
+import java.util.List;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsPageEvictionTest extends GridCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Test entry count. */
+ public static final int ENTRY_CNT = 1_000_000;
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ memCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+
+ memCfg.setPageSize(1024);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(50 * 1024 * 1024);
+ memPlcCfg.setMaxSize(50 * 1024 * 1024);
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ CacheConfiguration<DbKey, DbValue> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+ ccfg.setIndexedTypes(DbKey.class, DbValue.class);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+ cfg.setDiscoverySpi(
+ new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER)
+ );
+
+ cfg.setMarshaller(null);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ stopAllGrids();
+
+ startGrids(1);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageEvictionSql() throws Exception {
+ IgniteEx ig = grid(0);
+
+ try (IgniteDataStreamer<DbKey, DbValue> streamer = ig.dataStreamer(CACHE_NAME)) {
+ for (int i = 0; i < ENTRY_CNT; i++) {
+ streamer.addData(new DbKey(i), new DbValue(i, "value-" + i, Long.MAX_VALUE - i));
+
+ if (i > 0 && i % 10_000 == 0)
+ info("Done put: " + i);
+ }
+ }
+
+ IgniteCache<DbKey, DbValue> cache = ignite(0).cache(CACHE_NAME);
+
+ for (int i = 0; i < ENTRY_CNT; i++) {
+ assertEquals(Long.MAX_VALUE - i, cache.get(new DbKey(i)).lVal);
+
+ if (i > 0 && i % 10_000 == 0)
+ info("Done get: " + i);
+ }
+
+ for (int i = 0; i < ENTRY_CNT; i++) {
+ List<List<?>> rows = cache.query(
+ new SqlFieldsQuery("select lVal from DbValue where iVal=?").setArgs(i)
+ ).getAll();
+
+ assertEquals(1, rows.size());
+ assertEquals(Long.MAX_VALUE - i, rows.get(0).get(0));
+
+ if (i > 0 && i % 10_000 == 0)
+ info("Done SQL query: " + i);
+ }
+ }
+
+ /**
+ *
+ */
+ private static class DbKey implements Serializable {
+ /** */
+ private int val;
+
+ /**
+ * @param val Value.
+ */
+ private DbKey(int val) {
+ this.val = val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || !(o instanceof DbKey))
+ return false;
+
+ DbKey key = (DbKey)o;
+
+ return val == key.val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return val;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class DbValue implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String sVal;
+
+ /** */
+ @QuerySqlField
+ private long lVal;
+
+ /**
+ * @param iVal Integer value.
+ * @param sVal String value.
+ * @param lVal Long value.
+ */
+ private DbValue(int iVal, String sVal, long lVal) {
+ this.iVal = iVal;
+ this.sVal = sVal;
+ this.lVal = lVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DbValue dbVal = (DbValue)o;
+
+ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DbValue.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsRebalancingOnNotStableTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsRebalancingOnNotStableTopologyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsRebalancingOnNotStableTopologyTest.java
new file mode 100644
index 0000000..0724eb9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsRebalancingOnNotStableTopologyTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
+
+/**
+ * We start writing to unstable cluster.
+ * After that we start killing node.
+ * There will be entries in WAL which belongs to evicted partitions.
+ * We should ignore them (not throw exceptions). This point is tested.
+ */
+public class IgnitePdsRebalancingOnNotStableTopologyTest extends GridCommonAbstractTest {
+ /** Checkpoint frequency. */
+ private static final long CHECKPOINT_FREQUENCY = 2_000_000;
+
+ /** Cluster size. */
+ private static final int CLUSTER_SIZE = 5;
+
+ /** */
+ private static final String CACHE_NAME = "cache1";
+
+ /**
+ * @throws Exception When fails.
+ */
+ public void test() throws Exception {
+ Ignite ex = startGrid(0);
+
+ startGrid(1);
+
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch doneLatch = new CountDownLatch(1);
+
+ final Ignite ex1 = ex;
+
+ final AtomicBoolean stop = new AtomicBoolean();
+ final AtomicInteger keyCnt = new AtomicInteger();
+
+ Thread thread = new Thread(new Runnable() {
+ @Override public void run() {
+ ex1.active(true);
+
+ try {
+ checkTopology(2);
+
+ startLatch.countDown();
+
+ IgniteCache<Object, Object> cache1 = ex1.cache(CACHE_NAME);
+
+ int key = keyCnt.get();
+
+ while (!stop.get()) {
+ if (key > 0 && (key % 500 == 0)) {
+ U.sleep(5);
+
+ System.out.println("key = " + key);
+ }
+
+ cache1.put(key, -key);
+
+ key = keyCnt.incrementAndGet();
+ }
+ }
+ catch (Throwable th) {
+ th.printStackTrace();
+ }
+
+ doneLatch.countDown();
+ }
+ });
+
+ thread.setName("Data-Loader");
+ thread.start();
+
+ startLatch.await(60, TimeUnit.SECONDS);
+
+ for (int i = 2; i < CLUSTER_SIZE; i++) {
+ startGrid(i);
+
+ U.sleep(5000);
+ }
+
+ U.sleep(10000);
+
+ IgniteProcessProxy.kill("db.RebalancingOnNotStableTopologyTest2");
+
+ Thread.sleep(5000);
+
+ IgniteProcessProxy.kill("db.RebalancingOnNotStableTopologyTest1");
+
+ assert doneLatch.getCount() > 0;
+
+ stop.set(true);
+
+ doneLatch.await(600, TimeUnit.SECONDS);
+
+ IgniteProcessProxy.killAll();
+
+ stopAllGrids();
+
+ //start cluster. it will cause memory restoration and reading WAL.
+ ex = startGrids(CLUSTER_SIZE);
+
+ ex.active(true);
+
+ checkTopology(CLUSTER_SIZE);
+
+ IgniteCache<Object, Object> cache1 = ex.cache(CACHE_NAME);
+
+ assert keyCnt.get() > 0;
+
+ for (int i = 0; i < keyCnt.get(); i++)
+ assertEquals(-i, cache1.get(i));
+
+ System.out.println("Test finished with total keys count = " + keyCnt.get());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setActiveOnStart(false);
+
+ CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setCacheMode(CacheMode.PARTITIONED);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg.setBackups(2);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setCheckpointingFrequency(CHECKPOINT_FREQUENCY)
+ );
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(200 * 1024 * 1024);
+ memPlcCfg.setMaxSize(200 * 1024 * 1024);
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean isMultiJvm() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkTopology() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return TimeUnit.MINUTES.toMillis(10);
+ }
+}
[34/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
new file mode 100644
index 0000000..78d83b3
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTable.java
@@ -0,0 +1,556 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.GridPredicate3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiInClosure;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
+import static org.apache.ignite.IgniteSystemProperties.getFloat;
+
+/**
+ *
+ */
+public class FullPageIdTable {
+ /** Load factor. */
+ private static final float LOAD_FACTOR = getFloat(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, 2.5f);
+
+ /** */
+ private static final int BYTES_PER_ENTRY = /*cache ID*/4 + /*partition tag*/4 + /*page ID*/8 + /*pointer*/8;
+
+ /** */
+ private static final FullPageId EMPTY_FULL_PAGE_ID = new FullPageId(0, 0);
+
+ /** */
+ private static final long EMPTY_PAGE_ID = EMPTY_FULL_PAGE_ID.pageId();
+
+ /** */
+ private static final int EMPTY_CACHE_ID = EMPTY_FULL_PAGE_ID.cacheId();
+
+ /** */
+ private static final long REMOVED_PAGE_ID = 0x8000000000000000L;
+
+ /** */
+ private static final int REMOVED_CACHE_ID = 0;
+
+ /** */
+ private static final int EQUAL = 0;
+
+ /** */
+ private static final int EMPTY = 1;
+
+ /** */
+ private static final int REMOVED = -1;
+
+ /** */
+ private static final int NOT_EQUAL = 2;
+
+ /** */
+ private static final int OUTDATED = -3;
+
+ /** Max size, in elements. */
+ protected int capacity;
+
+ /** Maximum number of steps to try before failing. */
+ protected int maxSteps;
+
+ /** Pointer to the values array. */
+ protected long valPtr;
+
+ /**
+ * @return Estimated memory size required for this map to store the given number of elements.
+ */
+ public static long requiredMemory(long elementCnt) {
+ assert LOAD_FACTOR != 0;
+
+ return ((long)(elementCnt * LOAD_FACTOR)) * BYTES_PER_ENTRY + 8;
+ }
+
+ /**
+ * @param addr Base address.
+ * @param len Allocated memory length.
+ * @param clear If {@code true}, then memory is considered dirty and will be cleared. Otherwise,
+ * map will assume that the given memory region is in valid state.
+ */
+ public FullPageIdTable(long addr, long len, boolean clear) {
+ valPtr = addr;
+ capacity = (int)((len - 8) / BYTES_PER_ENTRY);
+
+ maxSteps = capacity;
+
+ if (clear)
+ clear();
+ }
+
+ /**
+ * @return Current number of entries in the map.
+ */
+ public final int size() {
+ return GridUnsafe.getInt(valPtr);
+ }
+
+ /**
+ * @return Maximum number of entries in the map. This maximum can not be always reached.
+ */
+ public final int capacity() {
+ return capacity;
+ }
+
+ /**
+ * Gets value associated with the given key.
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @return A value associated with the given key.
+ */
+ public long get(int cacheId, long pageId, int tag, long absent, long outdated) {
+ assert assertKey(cacheId, pageId);
+
+ int idx = getKey(cacheId, pageId, tag, false);
+
+ if (idx == -1)
+ return absent;
+
+ if (idx == -2)
+ return outdated;
+
+ return valueAt(idx);
+ }
+
+ /**
+ * Refresh outdated value.
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param tag Partition tag.
+ * @return A value associated with the given key.
+ */
+ public long refresh(int cacheId, long pageId, int tag) {
+ assert assertKey(cacheId, pageId);
+
+ int idx = getKey(cacheId, pageId, tag, true);
+
+ assert idx >= 0 : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
+ ", pageId=" + U.hexLong(pageId) + ']';
+
+ assert tagAt(idx) < tag : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
+ ", pageId=" + U.hexLong(pageId) + ", tagAtIdx=" + tagAt(idx) + ']';
+
+ setTagAt(idx, tag);
+
+ return valueAt(idx);
+ }
+
+ /**
+ * Associates the given key with the given value.
+ * @param cacheId Cache ID
+ * @param pageId Page ID.
+ * @param value Value to set.
+ */
+ public void put(int cacheId, long pageId, long value, int tag) {
+ assert assertKey(cacheId, pageId);
+
+ int index = putKey(cacheId, pageId, tag);
+
+ setValueAt(index, value);
+ }
+
+ /**
+ * Removes key-value association for the given key.
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ */
+ public void remove(int cacheId, long pageId, int tag) {
+ assert assertKey(cacheId, pageId);
+
+ int index = removeKey(cacheId, pageId, tag);
+
+ if (index >= 0)
+ setValueAt(index, 0);
+ }
+
+ /**
+ * Find nearest value from specified position to the right.
+ *
+ * @param idx Index to start searching from.
+ * @param absent Default value that will be returned if no values present.
+ * @return Closest value to the index and it's partition tag or {@code absent} and -1 if no values found.
+ */
+ public EvictCandidate getNearestAt(final int idx, final long absent) {
+ for (int i = idx; i < capacity + idx; i++) {
+ final int idx2 = i >= capacity ? i - capacity : i;
+
+ if (isValuePresentAt(idx2)) {
+ long base = entryBase(idx2);
+
+ int cacheId = GridUnsafe.getInt(base);
+ int tag = GridUnsafe.getInt(base + 4);
+ long pageId = GridUnsafe.getLong(base + 8);
+ long val = GridUnsafe.getLong(base + 16);
+
+ return new EvictCandidate(tag, val, new FullPageId(pageId, cacheId));
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * @param idx Index to clear value at.
+ * @param pred Test predicate.
+ * @param absent Value to return if the cell is empty.
+ * @return Value at the given index.
+ */
+ public long clearAt(int idx, GridPredicate3<Integer, Long, Integer> pred, long absent) {
+ long base = entryBase(idx);
+
+ int cacheId = GridUnsafe.getInt(base);
+ int tag = GridUnsafe.getInt(base + 4);
+ long pageId = GridUnsafe.getLong(base + 8);
+
+ if ((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
+ || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID))
+ return absent;
+
+ if (pred.apply(cacheId, pageId, tag)) {
+ long res = valueAt(idx);
+
+ setKeyAt(idx, REMOVED_CACHE_ID, REMOVED_PAGE_ID);
+ setValueAt(idx, 0);
+
+ return res;
+ }
+ else
+ return absent;
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @return Key index.
+ */
+ private int putKey(int cacheId, long pageId, int tag) {
+ int step = 1;
+
+ int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+
+ int foundIndex = -1;
+ int res;
+
+ do {
+ res = testKeyAt(index, cacheId, pageId, tag);
+
+ if (res == OUTDATED) {
+ foundIndex = index;
+
+ break;
+ }
+ else if (res == EMPTY) {
+ if (foundIndex == -1)
+ foundIndex = index;
+
+ break;
+ }
+ else if (res == REMOVED) {
+ // Must continue search to the first empty slot to ensure there are no duplicate mappings.
+ if (foundIndex == -1)
+ foundIndex = index;
+ }
+ else if (res == EQUAL)
+ return index;
+ else
+ assert res == NOT_EQUAL;
+
+ index++;
+
+ if (index >= capacity)
+ index -= capacity;
+ }
+ while (++step <= maxSteps);
+
+ if (foundIndex != -1) {
+ setKeyAt(foundIndex, cacheId, pageId);
+ setTagAt(foundIndex, tag);
+
+ if (res != OUTDATED)
+ incrementSize();
+
+ return foundIndex;
+ }
+
+ throw new IgniteOutOfMemoryException("No room for a new key");
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @return Key index.
+ */
+ private int getKey(int cacheId, long pageId, int tag, boolean refresh) {
+ int step = 1;
+
+ int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+
+ do {
+ long res = testKeyAt(index, cacheId, pageId, tag);
+
+ if (res == EQUAL)
+ return index;
+ else if (res == EMPTY)
+ return -1;
+ else if (res == OUTDATED)
+ return !refresh ? -2 : index;
+ else
+ assert res == REMOVED || res == NOT_EQUAL;
+
+ index++;
+
+ if (index >= capacity)
+ index -= capacity;
+ }
+ while (++step <= maxSteps);
+
+ return -1;
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @return Key index.
+ */
+ private int removeKey(int cacheId, long pageId, int tag) {
+ int step = 1;
+
+ int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+
+ int foundIndex = -1;
+
+ do {
+ long res = testKeyAt(index, cacheId, pageId, tag);
+
+ if (res == EQUAL || res == OUTDATED) {
+ foundIndex = index;
+
+ break;
+ }
+ else if (res == EMPTY)
+ return -1;
+ else
+ assert res == REMOVED || res == NOT_EQUAL;
+
+ index++;
+
+ if (index >= capacity)
+ index -= capacity;
+ }
+ while (++step <= maxSteps);
+
+ if (foundIndex != -1) {
+ setKeyAt(foundIndex, REMOVED_CACHE_ID, REMOVED_PAGE_ID);
+
+ decrementSize();
+ }
+
+ return foundIndex;
+ }
+
+ /**
+ * @param index Entry index.
+ * @return Key value.
+ */
+ private int testKeyAt(int index, int testCacheId, long testPageId, int testTag) {
+ long base = entryBase(index);
+
+ int cacheId = GridUnsafe.getInt(base);
+ int tag = GridUnsafe.getInt(base + 4);
+ long pageId = GridUnsafe.getLong(base + 8);
+
+ if (pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
+ return REMOVED;
+ else if (pageId == testPageId && cacheId == testCacheId && tag >= testTag)
+ return EQUAL;
+ else if (pageId == testPageId && cacheId == testCacheId && tag < testTag)
+ return OUTDATED;
+ else if (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID)
+ return EMPTY;
+ else
+ return NOT_EQUAL;
+ }
+
+ /**
+ * @param idx Index to test.
+ * @return {@code True} if value set for index.
+ */
+ private boolean isValuePresentAt(final int idx) {
+ long base = entryBase(idx);
+
+ int cacheId = GridUnsafe.getInt(base);
+ long pageId = GridUnsafe.getLong(base + 8);
+
+ return !((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
+ || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID));
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @return {@code True} if checks succeeded.
+ */
+ private boolean assertKey(int cacheId, long pageId) {
+ assert cacheId != EMPTY_CACHE_ID && PageIdUtils.isEffectivePageId(pageId):
+ "cacheId=" + cacheId + ", pageId=" + U.hexLong(pageId);
+
+ return true;
+ }
+
+ /**
+ * @param index Entry index.
+ * @param cacheId Cache ID to write.
+ * @param pageId Page ID to write.
+ */
+ private void setKeyAt(int index, int cacheId, long pageId) {
+ long base = entryBase(index);
+
+ GridUnsafe.putInt(base, cacheId);
+ GridUnsafe.putLong(base + 8, pageId);
+ }
+
+ /**
+ * Gets distance from the ideal key location to the actual location if this entry is present in the table,
+ * or returns negative distance that needed to be scanned to determine the absence of the mapping.
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param tag Tag.
+ * @return Distance scanned if the entry is found or negative distance scanned, if entry was not found.
+ */
+ public int distanceFromIdeal(int cacheId, long pageId, int tag) {
+ int step = 1;
+
+ int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+
+ int scans = 0;
+ boolean found = false;
+
+ do {
+ scans++;
+
+ long res = testKeyAt(index, cacheId, pageId, tag);
+
+ if (res == EQUAL || res == OUTDATED) {
+ found = true;
+
+ break;
+ }
+ else if (res == EMPTY)
+ break;
+ else
+ assert res == REMOVED || res == NOT_EQUAL;
+
+ index++;
+
+ if (index >= capacity)
+ index -= capacity;
+ }
+ while (++step <= maxSteps);
+
+ return found ? scans : -scans;
+ }
+
+ /**
+ * Scans all the elements in this table.
+ *
+ * @param visitor Visitor.
+ */
+ public void visitAll(IgniteBiInClosure<FullPageId, Long> visitor) {
+ for (int i = 0; i < capacity; i++) {
+ if (isValuePresentAt(i)) {
+ long base = entryBase(i);
+
+ int cacheId = GridUnsafe.getInt(base);
+ long pageId = GridUnsafe.getLong(base + 8);
+ long val = GridUnsafe.getLong(base + 16);
+
+ visitor.apply(new FullPageId(pageId, cacheId), val);
+ }
+ }
+ }
+
+ /**
+ * @param index Entry index.
+ * @return Value.
+ */
+ private long valueAt(int index) {
+ return GridUnsafe.getLong(entryBase(index) + 16);
+ }
+
+ /**
+ * @param index Entry index.
+ * @param value Value.
+ */
+ private void setValueAt(int index, long value) {
+ GridUnsafe.putLong(entryBase(index) + 16, value);
+ }
+
+ private long entryBase(int index) {
+ return valPtr + 8 + (long)index * BYTES_PER_ENTRY;
+ }
+
+ /**
+ * @param index Index to get tag for.
+ * @return Tag at the given index.
+ */
+ private int tagAt(int index) {
+ return GridUnsafe.getInt(entryBase(index) + 4);
+ }
+
+ /**
+ * @param index Index to set tag for.
+ * @param tag Tag to set at the given index.
+ */
+ private void setTagAt(int index, int tag) {
+ GridUnsafe.putInt(entryBase(index) + 4, tag);
+ }
+
+ /**
+ *
+ */
+ public void clear() {
+ GridUnsafe.setMemory(valPtr, (long)capacity * BYTES_PER_ENTRY + 8, (byte)0);
+ }
+
+ /**
+ *
+ */
+ private void incrementSize() {
+ GridUnsafe.putInt(valPtr, GridUnsafe.getInt(valPtr) + 1);
+ }
+
+ /**
+ *
+ */
+ private void decrementSize() {
+ GridUnsafe.putInt(valPtr, GridUnsafe.getInt(valPtr) - 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
new file mode 100644
index 0000000..3246f21
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryEx.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
+import org.apache.ignite.internal.util.lang.GridPredicate3;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public interface PageMemoryEx extends PageMemory {
+ /**
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param restore Determines if the page is locked for restore.
+ * @return ByteBuffer for modifying the page.
+ */
+ long writeLock(int cacheId, long pageId, long page, boolean restore);
+
+ /**
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param walPlc {@code True} if page should be recorded to WAL, {@code false} if the page must not
+ * be recorded and {@code null} for the default behavior.
+ * @param dirtyFlag Determines whether the page was modified since the last checkpoint.
+ * @param restore Determines if the page is locked for restore.
+ */
+ void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+ boolean dirtyFlag, boolean restore);
+
+ /**
+ * Gets or allocates metadata page for specified cacheId.
+ *
+ * @param cacheId Cache ID.
+ * @return Meta page for cacheId.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long metaPageId(int cacheId) throws IgniteCheckedException;
+
+ /**
+ * Gets or allocates partition metadata page for specified cacheId and partId.
+ *
+ * @param cacheId Cache ID.
+ * @param partId Partition ID.
+ * @return Meta page for cacheId and partId.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException;
+
+ /**
+ * @see #acquirePage(int, long)
+ * Will not read page from file if it is not present in memory
+ *
+ * @param cacheId Cache id.
+ * @param pageId Page id.
+ * @param restore Get page for restore
+ * @throws IgniteCheckedException If failed.
+ * @return Page.
+ */
+ public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException;
+
+ /**
+ * Heuristic method which allows a thread to check if it safe to start memory struture modifications
+ * in regard with checkpointing.
+ *
+ * @return {@code False} if there are too many dirty pages and a thread should wait for a
+ * checkpoint to begin.
+ */
+ public boolean safeToUpdate();
+
+ /**
+ * Gets a collection of dirty page IDs since the last checkpoint. If a dirty page is being written after
+ * the checkpointing operation begun, the modifications will be written to a temporary buffer which will
+ * be flushed to the main memory after the checkpointing finished. This method must be called when no
+ * concurrent operations on pages are performed.
+ *
+ * @return Collection of dirty page IDs.
+ * @throws IgniteException If checkpoint has been already started and was not finished.
+ */
+ public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException;
+
+ /**
+ * Finishes checkpoint operation.
+ */
+ public void finishCheckpoint();
+
+ /**
+ * Gets page byte buffer for the checkpoint procedure.
+ *
+ * @param pageId Page ID to get byte buffer for. The page ID must be present in the collection returned by
+ * the {@link #beginCheckpoint()} method call.
+ * @param tmpBuf Temporary buffer to write changes into.
+ * @param tracker Checkpoint metrics tracker.
+ * @return {@code True} if data were read, {@code false} otherwise (data already saved to storage).
+ * @throws IgniteException If failed to obtain page data.
+ */
+ @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker);
+
+ /**
+ * Marks partition as invalid / outdated.
+ *
+ * @param cacheId Cache ID.
+ * @param partId Partition ID.
+ * @return New partition tag.
+ */
+ public int invalidate(int cacheId, int partId);
+
+ /**
+ * Clears internal metadata of destroyed cache group.
+ *
+ * @param grpId Cache group ID.
+ */
+ public void onCacheGroupDestroyed(int grpId);
+
+ /**
+ * Asynchronously clears pages satisfying the given predicate.
+ *
+ * @param pred Predicate for cache group id, pageId and partition tag.
+ * @param cleanDirty Flag indicating that dirty pages collection should be cleaned.
+ * @return Future that will be completed when all pages are cleared.
+ */
+ public IgniteInternalFuture<Void> clearAsync(GridPredicate3<Integer, Long, Integer> pred, boolean cleanDirty);
+}
[35/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
new file mode 100644
index 0000000..eb5df6b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java
@@ -0,0 +1,1482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageSetFreeListPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListInitNewPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListRemovePageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetNextRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousRecord;
+import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.GridArrays;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.getPageId;
+
+/**
+ * Striped doubly-linked list of page IDs optionally organized in buckets.
+ */
+public abstract class PagesList extends DataStructure {
+ /** */
+ private static final int TRY_LOCK_ATTEMPTS =
+ IgniteSystemProperties.getInteger("IGNITE_PAGES_LIST_TRY_LOCK_ATTEMPTS", 10);
+
+ /** */
+ private static final int MAX_STRIPES_PER_BUCKET =
+ IgniteSystemProperties.getInteger("IGNITE_PAGES_LIST_STRIPES_PER_BUCKET",
+ Math.min(8, Runtime.getRuntime().availableProcessors() * 2));
+
+ /** */
+ protected final AtomicLong[] bucketsSize;
+
+ /** */
+ protected volatile boolean changed;
+
+ /** Page ID to store list metadata. */
+ private final long metaPageId;
+
+ /** Number of buckets. */
+ private final int buckets;
+
+ /** Name (for debug purposes). */
+ protected final String name;
+
+ /** */
+ private final PageHandler<Void, Boolean> cutTail = new CutTail();
+
+ /**
+ *
+ */
+ private final class CutTail extends PageHandler<Void, Boolean> {
+ @Override public Boolean run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ Void ignore,
+ int bucket) throws IgniteCheckedException {
+ assert getPageId(pageAddr) == pageId;
+
+ PagesListNodeIO io = (PagesListNodeIO)iox;
+
+ long tailId = io.getNextId(pageAddr);
+
+ assert tailId != 0;
+
+ io.setNextId(pageAddr, 0L);
+
+ if (needWalDeltaRecord(pageId, page, walPlc))
+ wal.log(new PagesListSetNextRecord(cacheId, pageId, 0L));
+
+ updateTail(bucket, tailId, pageId);
+
+ return TRUE;
+ }
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param name Name (for debug purpose).
+ * @param pageMem Page memory.
+ * @param buckets Number of buckets.
+ * @param wal Write ahead log manager.
+ * @param metaPageId Metadata page ID.
+ */
+ protected PagesList(
+ int cacheId,
+ String name,
+ PageMemory pageMem,
+ int buckets,
+ IgniteWriteAheadLogManager wal,
+ long metaPageId
+ ) {
+ super(cacheId, pageMem, wal);
+
+ this.name = name;
+ this.buckets = buckets;
+ this.metaPageId = metaPageId;
+
+ bucketsSize = new AtomicLong[buckets];
+
+ for (int i = 0; i < buckets; i++)
+ bucketsSize[i] = new AtomicLong();
+ }
+
+ /**
+ * @param metaPageId Metadata page ID.
+ * @param initNew {@code True} if new list if created, {@code false} if should be initialized from metadata.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final void init(long metaPageId, boolean initNew) throws IgniteCheckedException {
+ if (metaPageId != 0L) {
+ if (initNew) {
+ init(metaPageId, PagesListMetaIO.VERSIONS.latest());
+ }
+ else {
+ Map<Integer, GridLongList> bucketsData = new HashMap<>();
+
+ long nextId = metaPageId;
+
+ while (nextId != 0) {
+ final long pageId = nextId;
+ final long page = acquirePage(pageId);
+
+ try {
+ long pageAddr = readLock(pageId, page); // No concurrent recycling on init.
+
+ assert pageAddr != 0L;
+
+ try {
+ PagesListMetaIO io = PagesListMetaIO.VERSIONS.forPage(pageAddr);
+
+ io.getBucketsData(pageAddr, bucketsData);
+
+ nextId = io.getNextMetaPageId(pageAddr);
+
+ assert nextId != pageId :
+ "Loop detected [next=" + U.hexLong(nextId) + ", cur=" + U.hexLong(pageId) + ']';
+
+
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ for (Map.Entry<Integer, GridLongList> e : bucketsData.entrySet()) {
+ int bucket = e.getKey();
+ long bucketSize = 0;
+
+ Stripe[] old = getBucket(bucket);
+ assert old == null;
+
+ long[] upd = e.getValue().array();
+
+ Stripe[] tails = new Stripe[upd.length];
+
+ for (int i = 0; i < upd.length; i++) {
+ long tailId = upd[i];
+
+ long prevId = tailId;
+ int cnt = 0;
+
+ while (prevId != 0L) {
+ final long pageId = prevId;
+ final long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page);
+
+ assert pageAddr != 0L;
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
+
+ cnt += io.getCount(pageAddr);
+ prevId = io.getPreviousId(pageAddr);
+
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket) && prevId != 0L)
+ cnt++;
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ Stripe stripe = new Stripe(tailId, cnt == 0);
+ tails[i] = stripe;
+ bucketSize += cnt;
+ }
+
+ boolean ok = casBucket(bucket, null, tails);
+ assert ok;
+
+ bucketsSize[bucket].set(bucketSize);
+
+ changed = true;
+ }
+ }
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void saveMetadata() throws IgniteCheckedException {
+ assert metaPageId != 0;
+
+ long curId = 0L;
+ long curPage = 0L;
+ long curAddr = 0L;
+
+ PagesListMetaIO curIo = null;
+
+ long nextPageId = metaPageId;
+
+ if (!changed)
+ return;
+
+ try {
+ for (int bucket = 0; bucket < buckets; bucket++) {
+ Stripe[] tails = getBucket(bucket);
+
+ if (tails != null) {
+ int tailIdx = 0;
+
+ while (tailIdx < tails.length) {
+ int written = curPage != 0L ? curIo.addTails(pageMem.pageSize(), curAddr, bucket, tails, tailIdx) : 0;
+
+ if (written == 0) {
+ if (nextPageId == 0L) {
+ nextPageId = allocatePageNoReuse();
+
+ if (curPage != 0L) {
+ curIo.setNextMetaPageId(curAddr, nextPageId);
+
+ releaseAndClose(curId, curPage, curAddr);
+ }
+
+ curId = nextPageId;
+ curPage = acquirePage(curId);
+ curAddr = writeLock(curId, curPage);
+
+ curIo = PagesListMetaIO.VERSIONS.latest();
+
+ curIo.initNewPage(curAddr, curId, pageSize());
+ }
+ else {
+ releaseAndClose(curId, curPage, curAddr);
+
+ curId = nextPageId;
+ curPage = acquirePage(curId);
+ curAddr = writeLock(curId, curPage);
+
+ curIo = PagesListMetaIO.VERSIONS.forPage(curAddr);
+
+ curIo.resetCount(curAddr);
+ }
+
+ nextPageId = curIo.getNextMetaPageId(curAddr);
+ }
+ else
+ tailIdx += written;
+ }
+ }
+ }
+ }
+ finally {
+ releaseAndClose(curId, curPage, curAddr);
+ }
+
+ while (nextPageId != 0L) {
+ long pageId = nextPageId;
+
+ long page = acquirePage(pageId);
+ try {
+ long pageAddr = writeLock(pageId, page);
+
+ try {
+ PagesListMetaIO io = PagesListMetaIO.VERSIONS.forPage(pageAddr);
+
+ io.resetCount(pageAddr);
+
+ if (needWalDeltaRecord(pageId, page, null))
+ wal.log(new PageListMetaResetCountRecord(cacheId, pageId));
+
+ nextPageId = io.getNextMetaPageId(pageAddr);
+ }
+ finally {
+ writeUnlock(pageId, page, pageAddr, true);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ changed = false;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page absolute pointer.
+ * @param pageAddr Page address.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void releaseAndClose(long pageId, long page, long pageAddr) throws IgniteCheckedException {
+ if (page != 0L) {
+ try {
+ // No special WAL record because we most likely changed the whole page.
+ writeUnlock(pageId, page, pageAddr, TRUE, true);
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+ }
+
+ /**
+ * @param bucket Bucket index.
+ * @return Bucket.
+ */
+ protected abstract Stripe[] getBucket(int bucket);
+
+ /**
+ * @param bucket Bucket index.
+ * @param exp Expected bucket.
+ * @param upd Updated bucket.
+ * @return {@code true} If succeeded.
+ */
+ protected abstract boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd);
+
+ /**
+ * @param bucket Bucket index.
+ * @return {@code true} If it is a reuse bucket.
+ */
+ protected abstract boolean isReuseBucket(int bucket);
+
+ /**
+ * @param io IO.
+ * @param prevId Previous page ID.
+ * @param prev Previous page buffer.
+ * @param nextId Next page ID.
+ * @param next Next page buffer.
+ */
+ private void setupNextPage(PagesListNodeIO io, long prevId, long prev, long nextId, long next) {
+ assert io.getNextId(prev) == 0L;
+
+ io.initNewPage(next, nextId, pageSize());
+ io.setPreviousId(next, prevId);
+
+ io.setNextId(prev, nextId);
+ }
+
+ /**
+ * Adds stripe to the given bucket.
+ *
+ * @param bucket Bucket.
+ * @param reuse {@code True} if possible to use reuse list.
+ * @throws IgniteCheckedException If failed.
+ * @return Tail page ID.
+ */
+ private Stripe addStripe(int bucket, boolean reuse) throws IgniteCheckedException {
+ long pageId = reuse ? allocatePage(null) : allocatePageNoReuse();
+
+ init(pageId, PagesListNodeIO.VERSIONS.latest());
+
+ Stripe stripe = new Stripe(pageId, true);
+
+ for (;;) {
+ Stripe[] old = getBucket(bucket);
+ Stripe[] upd;
+
+ if (old != null) {
+ int len = old.length;
+
+ upd = Arrays.copyOf(old, len + 1);
+
+ upd[len] = stripe;
+ }
+ else
+ upd = new Stripe[]{stripe};
+
+ if (casBucket(bucket, old, upd))
+ return stripe;
+ }
+ }
+
+ /**
+ * @param bucket Bucket index.
+ * @param oldTailId Old tail page ID to replace.
+ * @param newTailId New tail page ID.
+ * @return {@code True} if stripe was removed.
+ */
+ private boolean updateTail(int bucket, long oldTailId, long newTailId) {
+ int idx = -1;
+
+ for (;;) {
+ Stripe[] tails = getBucket(bucket);
+
+ // Tail must exist to be updated.
+ assert !F.isEmpty(tails) : "Missing tails [bucket=" + bucket + ", tails=" + Arrays.toString(tails) +
+ ", metaPage=" + U.hexLong(metaPageId) + ']';
+
+ idx = findTailIndex(tails, oldTailId, idx);
+
+ assert tails[idx].tailId == oldTailId;
+
+ if (newTailId == 0L) {
+ if (tails.length <= MAX_STRIPES_PER_BUCKET / 2) {
+ tails[idx].empty = true;
+
+ return false;
+ }
+
+ Stripe[] newTails;
+
+ if (tails.length != 1)
+ newTails = GridArrays.remove(tails, idx);
+ else
+ newTails = null; // Drop the bucket completely.
+
+ if (casBucket(bucket, tails, newTails))
+ return true;
+ }
+ else {
+ // It is safe to assign new tail since we do it only when write lock on tail is held.
+ tails[idx].tailId = newTailId;
+
+ return true;
+ }
+ }
+ }
+
+ /**
+ * @param tails Tails.
+ * @param tailId Tail ID to find.
+ * @param expIdx Expected index.
+ * @return First found index of the given tail ID.
+ */
+ private static int findTailIndex(Stripe[] tails, long tailId, int expIdx) {
+ if (expIdx != -1 && tails.length > expIdx && tails[expIdx].tailId == tailId)
+ return expIdx;
+
+ for (int i = 0; i < tails.length; i++) {
+ if (tails[i].tailId == tailId)
+ return i;
+ }
+
+ throw new IllegalStateException("Tail not found: " + tailId);
+ }
+
+ /**
+ * @param bucket Bucket.
+ * @return Page ID where the given page
+ * @throws IgniteCheckedException If failed.
+ */
+ private Stripe getPageForPut(int bucket) throws IgniteCheckedException {
+ Stripe[] tails = getBucket(bucket);
+
+ if (tails == null)
+ return addStripe(bucket, true);
+
+ return randomTail(tails);
+ }
+
+ /**
+ * @param tails Tails.
+ * @return Random tail.
+ */
+ private static Stripe randomTail(Stripe[] tails) {
+ int len = tails.length;
+
+ assert len != 0;
+
+ return tails[randomInt(len)];
+ }
+
+ /**
+ * !!! For tests only, does not provide any correctness guarantees for concurrent access.
+ *
+ * @param bucket Bucket index.
+ * @return Number of pages stored in this list.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final long storedPagesCount(int bucket) throws IgniteCheckedException {
+ long res = 0;
+
+ Stripe[] tails = getBucket(bucket);
+
+ if (tails != null) {
+ for (Stripe tail : tails) {
+ long tailId = tail.tailId;
+
+ while (tailId != 0L) {
+ final long pageId = tailId;
+ final long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page);
+
+ assert pageAddr != 0L;
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
+
+ int cnt = io.getCount(pageAddr);
+
+ assert cnt >= 0;
+
+ res += cnt;
+ tailId = io.getPreviousId(pageAddr);
+
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket) && tailId != 0L)
+ res++;
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+ }
+ }
+
+ assert res == bucketsSize[bucket].get() : "Wrong bucket size counter [exp=" + res + ", cntr=" + bucketsSize[bucket].get() + ']';
+
+ return res;
+ }
+
+ /**
+ * @param bag Reuse bag.
+ * @param dataId Data page ID.
+ * @param dataPage Data page pointer.
+ * @param dataAddr Data page address.
+ * @param bucket Bucket.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final void put(
+ ReuseBag bag,
+ final long dataId,
+ final long dataPage,
+ final long dataAddr,
+ int bucket)
+ throws IgniteCheckedException {
+ assert bag == null ^ dataAddr == 0L;
+
+ for (int lockAttempt = 0; ;) {
+ Stripe stripe = getPageForPut(bucket);
+
+ final long tailId = stripe.tailId;
+ final long tailPage = acquirePage(tailId);
+
+ try {
+ long tailAddr = writeLockPage(tailId, tailPage, bucket, lockAttempt++); // Explicit check.
+
+ if (tailAddr == 0L) {
+ if (isReuseBucket(bucket) && lockAttempt == TRY_LOCK_ATTEMPTS)
+ addStripeForReuseBucket(bucket);
+
+ continue;
+ }
+
+ assert PageIO.getPageId(tailAddr) == tailId : "pageId = " + PageIO.getPageId(tailAddr) + ", tailId = " + tailId;
+ assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE;
+
+ boolean ok = false;
+
+ try {
+ PagesListNodeIO io = PageIO.getPageIO(tailAddr);
+
+ ok = bag != null ?
+ // Here we can always take pages from the bag to build our list.
+ putReuseBag(tailId, tailPage, tailAddr, io, bag, bucket) :
+ // Here we can use the data page to build list only if it is empty and
+ // it is being put into reuse bucket. Usually this will be true, but there is
+ // a case when there is no reuse bucket in the free list, but then deadlock
+ // on node page allocation from separate reuse list is impossible.
+ // If the data page is not empty it can not be put into reuse bucket and thus
+ // the deadlock is impossible as well.
+ putDataPage(tailId, tailPage, tailAddr, io, dataId, dataPage, dataAddr, bucket);
+
+ if (ok) {
+ stripe.empty = false;
+
+ return;
+ }
+ }
+ finally {
+ writeUnlock(tailId, tailPage, tailAddr, ok);
+ }
+ }
+ finally {
+ releasePage(tailId, tailPage);
+ }
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param dataId Data page ID.
+ * @param dataPage Data page pointer.
+ * @param dataAddr Data page address.
+ * @param bucket Bucket.
+ * @return {@code true} If succeeded.
+ * @throws IgniteCheckedException If failed.
+ */
+ private boolean putDataPage(
+ final long pageId,
+ final long page,
+ final long pageAddr,
+ PagesListNodeIO io,
+ final long dataId,
+ final long dataPage,
+ final long dataAddr,
+ int bucket
+ ) throws IgniteCheckedException {
+ if (io.getNextId(pageAddr) != 0L)
+ return false; // Splitted.
+
+ int idx = io.addPage(pageAddr, dataId, pageSize());
+
+ if (idx == -1)
+ handlePageFull(pageId, page, pageAddr, io, dataId, dataPage, dataAddr, bucket);
+ else {
+ incrementBucketSize(bucket);
+
+ if (needWalDeltaRecord(pageId, page, null))
+ wal.log(new PagesListAddPageRecord(cacheId, pageId, dataId));
+
+ DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
+ dataIO.setFreeListPageId(dataAddr, pageId);
+
+ if (needWalDeltaRecord(dataId, dataPage, null))
+ wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, pageId));
+ }
+
+ return true;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param dataId Data page ID.
+ * @param data Data page pointer.
+ * @param dataAddr Data page address.
+ * @param bucket Bucket index.
+ * @throws IgniteCheckedException If failed.
+ * */
+ private void handlePageFull(
+ final long pageId,
+ final long page,
+ final long pageAddr,
+ PagesListNodeIO io,
+ final long dataId,
+ final long data,
+ final long dataAddr,
+ int bucket
+ ) throws IgniteCheckedException {
+ DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
+
+ // Attempt to add page failed: the node page is full.
+ if (isReuseBucket(bucket)) {
+ // If we are on the reuse bucket, we can not allocate new page, because it may cause deadlock.
+ assert dataIO.isEmpty(dataAddr); // We can put only empty data pages to reuse bucket.
+
+ // Change page type to index and add it as next node page to this list.
+ long newDataId = PageIdUtils.changeType(dataId, FLAG_IDX);
+
+ setupNextPage(io, pageId, pageAddr, newDataId, dataAddr);
+
+ if (needWalDeltaRecord(pageId, page, null))
+ wal.log(new PagesListSetNextRecord(cacheId, pageId, newDataId));
+
+ if (needWalDeltaRecord(dataId, data, null))
+ wal.log(new PagesListInitNewPageRecord(
+ cacheId,
+ dataId,
+ io.getType(),
+ io.getVersion(),
+ newDataId,
+ pageId, 0L));
+
+ // In reuse bucket the page itself can be used as a free page.
+ incrementBucketSize(bucket);
+
+ updateTail(bucket, pageId, newDataId);
+ }
+ else {
+ // Just allocate a new node page and add our data page there.
+ final long nextId = allocatePage(null);
+ final long nextPage = acquirePage(nextId);
+
+ try {
+ long nextPageAddr = writeLock(nextId, nextPage); // Newly allocated page.
+
+ assert nextPageAddr != 0L;
+
+ // Here we should never write full page, because it is known to be new.
+ Boolean nextWalPlc = FALSE;
+
+ try {
+ setupNextPage(io, pageId, pageAddr, nextId, nextPageAddr);
+
+ if (needWalDeltaRecord(pageId, page, null))
+ wal.log(new PagesListSetNextRecord(cacheId, pageId, nextId));
+
+ int idx = io.addPage(nextPageAddr, dataId, pageSize());
+
+ if (needWalDeltaRecord(nextId, nextPage, nextWalPlc))
+ wal.log(new PagesListInitNewPageRecord(
+ cacheId,
+ nextId,
+ io.getType(),
+ io.getVersion(),
+ nextId,
+ pageId,
+ dataId
+ ));
+
+ assert idx != -1;
+
+ dataIO.setFreeListPageId(dataAddr, nextId);
+
+ if (needWalDeltaRecord(dataId, data, null))
+ wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, nextId));
+
+ incrementBucketSize(bucket);
+
+ updateTail(bucket, pageId, nextId);
+ }
+ finally {
+ writeUnlock(nextId, nextPage, nextPageAddr, nextWalPlc, true);
+ }
+ }
+ finally {
+ releasePage(nextId, nextPage);
+ }
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param bag Reuse bag.
+ * @param bucket Bucket.
+ * @return {@code true} If succeeded.
+ * @throws IgniteCheckedException if failed.
+ */
+ @SuppressWarnings("ForLoopReplaceableByForEach")
+ private boolean putReuseBag(
+ final long pageId,
+ final long page,
+ final long pageAddr,
+ PagesListNodeIO io,
+ ReuseBag bag,
+ int bucket
+ ) throws IgniteCheckedException {
+ if (io.getNextId(pageAddr) != 0L)
+ return false; // Splitted.
+
+ long nextId;
+
+ long prevId = pageId;
+ long prevPage = page;
+ long prevAddr = pageAddr;
+
+ Boolean walPlc = null;
+
+ GridLongList locked = null; // TODO may be unlock right away and do not keep all these pages locked?
+
+ try {
+ while ((nextId = bag.pollFreePage()) != 0L) {
+ int idx = io.addPage(prevAddr, nextId, pageSize());
+
+ if (idx == -1) { // Attempt to add page failed: the node page is full.
+
+ final long nextPage = acquirePage(nextId);
+
+ try {
+ long nextPageAddr = writeLock(nextId, nextPage); // Page from reuse bag can't be concurrently recycled.
+
+ assert nextPageAddr != 0L;
+
+ if (locked == null) {
+ locked = new GridLongList(6);
+ }
+
+ locked.add(nextId);
+ locked.add(nextPage);
+ locked.add(nextPageAddr);
+
+ setupNextPage(io, prevId, prevAddr, nextId, nextPageAddr);
+
+ if (needWalDeltaRecord(prevId, prevPage, walPlc))
+ wal.log(new PagesListSetNextRecord(cacheId, prevId, nextId));
+
+ // Here we should never write full page, because it is known to be new.
+ if (needWalDeltaRecord(nextId, nextPage, FALSE))
+ wal.log(new PagesListInitNewPageRecord(
+ cacheId,
+ nextId,
+ io.getType(),
+ io.getVersion(),
+ nextId,
+ prevId,
+ 0L
+ ));
+
+ // In reuse bucket the page itself can be used as a free page.
+ if (isReuseBucket(bucket))
+ incrementBucketSize(bucket);
+
+ // Switch to this new page, which is now a part of our list
+ // to add the rest of the bag to the new page.
+ prevAddr = nextPageAddr;
+ prevId = nextId;
+ prevPage = nextPage;
+ // Starting from tis point all wal records are written for reused pages from the bag.
+ // This mean that we use delta records only.
+ walPlc = FALSE;
+ }
+ finally {
+ releasePage(nextId, nextPage);
+ }
+ }
+ else {
+ // TODO: use single WAL record for bag?
+ if (needWalDeltaRecord(prevId, prevPage, walPlc))
+ wal.log(new PagesListAddPageRecord(cacheId, prevId, nextId));
+
+ incrementBucketSize(bucket);
+ }
+ }
+ }
+ finally {
+ if (locked != null) {
+ // We have to update our bucket with the new tail.
+ updateTail(bucket, pageId, prevId);
+
+ // Release write.
+ for (int i = 0; i < locked.size(); i+=3) {
+ writeUnlock(locked.get(i), locked.get(i + 1), locked.get(i + 2), FALSE, true);
+ }
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * @param bucket Bucket index.
+ * @return Page for take.
+ */
+ private Stripe getPageForTake(int bucket) {
+ Stripe[] tails = getBucket(bucket);
+
+ if (tails == null || bucketsSize[bucket].get() == 0)
+ return null;
+
+ int len = tails.length;
+ int init = randomInt(len);
+ int cur = init;
+
+ while (true) {
+ Stripe stripe = tails[cur];
+
+ if (!stripe.empty)
+ return stripe;
+
+ if ((cur = (cur + 1) % len) == init)
+ return null;
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param bucket Bucket.
+ * @param lockAttempt Lock attempts counter.
+ * @return Page address if page is locked of {@code null} if can retry lock.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long writeLockPage(long pageId, long page, int bucket, int lockAttempt)
+ throws IgniteCheckedException {
+ long pageAddr = tryWriteLock(pageId, page);
+
+ if (pageAddr != 0L)
+ return pageAddr;
+
+ if (lockAttempt == TRY_LOCK_ATTEMPTS) {
+ Stripe[] stripes = getBucket(bucket);
+
+ if (stripes == null || stripes.length < MAX_STRIPES_PER_BUCKET) {
+ if (!isReuseBucket(bucket))
+ addStripe(bucket, true);
+
+ return 0L;
+ }
+ }
+
+ return lockAttempt < TRY_LOCK_ATTEMPTS ? 0L : writeLock(pageId, page); // Must be explicitly checked further.
+ }
+
+ /**
+ * @param bucket Bucket.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void addStripeForReuseBucket(int bucket) throws IgniteCheckedException {
+ assert isReuseBucket(bucket);
+
+ Stripe[] stripes = getBucket(bucket);
+
+ if (stripes == null || stripes.length < MAX_STRIPES_PER_BUCKET)
+ addStripe(bucket, false);
+ }
+
+ /**
+ * @param bucket Bucket index.
+ * @param initIoVers Optional IO to initialize page.
+ * @return Removed page ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers) throws IgniteCheckedException {
+ for (int lockAttempt = 0; ;) {
+ Stripe stripe = getPageForTake(bucket);
+
+ if (stripe == null)
+ return 0L;
+
+ final long tailId = stripe.tailId;
+ final long tailPage = acquirePage(tailId);
+
+ try {
+ long tailAddr = writeLockPage(tailId, tailPage, bucket, lockAttempt++); // Explicit check.
+
+ if (tailAddr == 0L) {
+ if (isReuseBucket(bucket) && lockAttempt == TRY_LOCK_ATTEMPTS)
+ addStripeForReuseBucket(bucket);
+
+ continue;
+ }
+
+ if (stripe.empty) {
+ // Another thread took the last page.
+ writeUnlock(tailId, tailPage, tailAddr, false);
+
+ if (bucketsSize[bucket].get() > 0) {
+ lockAttempt--; // Ignore current attempt.
+
+ continue;
+ }
+ else
+ return 0L;
+ }
+
+ assert PageIO.getPageId(tailAddr) == tailId : "tailId = " + tailId + ", tailPageId = " + PageIO.getPageId(tailAddr);
+ assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE;
+
+ boolean dirty = false;
+ long dataPageId;
+ long recycleId = 0L;
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailAddr);
+
+ if (io.getNextId(tailAddr) != 0) {
+ // It is not a tail anymore, retry.
+ continue;
+ }
+
+ long pageId = io.takeAnyPage(tailAddr);
+
+ if (pageId != 0L) {
+ decrementBucketSize(bucket);
+
+ if (needWalDeltaRecord(tailId, tailPage, null))
+ wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId));
+
+ dirty = true;
+
+ dataPageId = pageId;
+
+ if (io.isEmpty(tailAddr)) {
+ long prevId = io.getPreviousId(tailAddr);
+
+ // If we got an empty page in non-reuse bucket, move it back to reuse list
+ // to prevent empty page leak to data pages.
+ if (!isReuseBucket(bucket)) {
+ if (prevId != 0L) {
+ Boolean ok = write(prevId, cutTail, null, bucket, FALSE);
+
+ assert ok == TRUE : ok;
+
+ recycleId = recyclePage(tailId, tailPage, tailAddr, null);
+ }
+ else
+ stripe.empty = true;
+ }
+ else
+ stripe.empty = prevId == 0L;
+ }
+ }
+ else {
+ // The tail page is empty, but stripe is not. It might
+ // happen only if we are in reuse bucket and it has
+ // a previous page, so, the current page can be collected
+ assert isReuseBucket(bucket);
+
+ long prevId = io.getPreviousId(tailAddr);
+
+ assert prevId != 0L;
+
+ Boolean ok = write(prevId, cutTail, bucket, FALSE);
+
+ assert ok == TRUE : ok;
+
+ decrementBucketSize(bucket);
+
+ if (initIoVers != null) {
+ dataPageId = PageIdUtils.changeType(tailId, FLAG_DATA);
+
+ PageIO initIo = initIoVers.latest();
+
+ initIo.initNewPage(tailAddr, dataPageId, pageSize());
+
+ if (needWalDeltaRecord(tailId, tailPage, null)) {
+ wal.log(new InitNewPageRecord(cacheId, tailId, initIo.getType(),
+ initIo.getVersion(), dataPageId));
+ }
+ }
+ else
+ dataPageId = recyclePage(tailId, tailPage, tailAddr, null);
+
+ dirty = true;
+ }
+
+ // If we do not have a previous page (we are at head), then we still can return
+ // current page but we have to drop the whole stripe. Since it is a reuse bucket,
+ // we will not do that, but just return 0L, because this may produce contention on
+ // meta page.
+ }
+ finally {
+ writeUnlock(tailId, tailPage, tailAddr, dirty);
+ }
+
+ // Put recycled page (if any) to the reuse bucket after tail is unlocked.
+ if (recycleId != 0L) {
+ assert !isReuseBucket(bucket);
+
+ reuseList.addForRecycle(new SingletonReuseBag(recycleId));
+ }
+
+ return dataPageId;
+ }
+ finally {
+ releasePage(tailId, tailPage);
+ }
+ }
+ }
+
+ /**
+ * @param dataId Data page ID.
+ * @param dataPage Data page pointer.
+ * @param dataAddr Data page address.
+ * @param dataIO Data page IO.
+ * @param bucket Bucket index.
+ * @throws IgniteCheckedException If failed.
+ * @return {@code True} if page was removed.
+ */
+ protected final boolean removeDataPage(
+ final long dataId,
+ final long dataPage,
+ final long dataAddr,
+ DataPageIO dataIO,
+ int bucket)
+ throws IgniteCheckedException {
+ final long pageId = dataIO.getFreeListPageId(dataAddr);
+
+ assert pageId != 0;
+
+ final long page = acquirePage(pageId);
+ try {
+ long nextId;
+
+ long recycleId = 0L;
+
+ long pageAddr = writeLock(pageId, page); // Explicit check.
+
+ if (pageAddr == 0L)
+ return false;
+
+ boolean rmvd = false;
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
+
+ rmvd = io.removePage(pageAddr, dataId);
+
+ if (!rmvd)
+ return false;
+
+ decrementBucketSize(bucket);
+
+ if (needWalDeltaRecord(pageId, page, null))
+ wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataId));
+
+ // Reset free list page ID.
+ dataIO.setFreeListPageId(dataAddr, 0L);
+
+ if (needWalDeltaRecord(dataId, dataPage, null))
+ wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, 0L));
+
+ if (!io.isEmpty(pageAddr))
+ return true; // In optimistic case we still have something in the page and can leave it as is.
+
+ // If the page is empty, we have to try to drop it and link next and previous with each other.
+ nextId = io.getNextId(pageAddr);
+
+ // If there are no next page, then we can try to merge without releasing current write lock,
+ // because if we will need to lock previous page, the locking order will be already correct.
+ if (nextId == 0L) {
+ long prevId = io.getPreviousId(pageAddr);
+
+ recycleId = mergeNoNext(pageId, page, pageAddr, prevId, bucket);
+ }
+ }
+ finally {
+ writeUnlock(pageId, page, pageAddr, rmvd);
+ }
+
+ // Perform a fair merge after lock release (to have a correct locking order).
+ if (nextId != 0L)
+ recycleId = merge(pageId, page, nextId, bucket);
+
+ if (recycleId != 0L)
+ reuseList.addForRecycle(new SingletonReuseBag(recycleId));
+
+ return true;
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param prevId Previous page ID.
+ * @param bucket Bucket index.
+ * @return Page ID to recycle.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long mergeNoNext(
+ long pageId,
+ long page,
+ long pageAddr,
+ long prevId,
+ int bucket)
+ throws IgniteCheckedException {
+ // If we do not have a next page (we are tail) and we are on reuse bucket,
+ // then we can leave as is as well, because it is normal to have an empty tail page here.
+ if (isReuseBucket(bucket))
+ return 0L;
+
+ if (prevId != 0L) { // Cut tail if we have a previous page.
+ Boolean ok = write(prevId, cutTail, null, bucket, FALSE);
+
+ assert ok == TRUE: ok;
+ }
+ else {
+ // If we don't have a previous, then we are tail page of free list, just drop the stripe.
+ boolean rmvd = updateTail(bucket, pageId, 0L);
+
+ if (!rmvd)
+ return 0L;
+ }
+
+ return recyclePage(pageId, page, pageAddr, null);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param nextId Next page ID.
+ * @param bucket Bucket index.
+ * @return Page ID to recycle.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long merge(
+ final long pageId,
+ final long page,
+ long nextId,
+ int bucket)
+ throws IgniteCheckedException {
+ assert nextId != 0; // We should do mergeNoNext then.
+
+ // Lock all the pages in correct order (from next to previous) and do the merge in retry loop.
+ for (;;) {
+ final long curId = nextId;
+ final long curPage = curId == 0L ? 0L : acquirePage(curId);
+ try {
+ boolean write = false;
+
+ final long curAddr = curPage == 0L ? 0L : writeLock(curId, curPage); // Explicit check.
+ final long pageAddr = writeLock(pageId, page); // Explicit check.
+
+ if (pageAddr == 0L) {
+ if (curAddr != 0L) // Unlock next page if needed.
+ writeUnlock(curId, curPage, curAddr, false);
+
+ return 0L; // Someone has merged or taken our empty page concurrently. Nothing to do here.
+ }
+
+ try {
+ PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
+
+ if (!io.isEmpty(pageAddr))
+ return 0L; // No need to merge anymore.
+
+ // Check if we see a consistent state of the world.
+ if (io.getNextId(pageAddr) == curId && (curId == 0L) == (curAddr == 0L)) {
+ long recycleId = doMerge(pageId, page, pageAddr, io, curId, curPage, curAddr, bucket);
+
+ write = true;
+
+ return recycleId; // Done.
+ }
+
+ // Reread next page ID and go for retry.
+ nextId = io.getNextId(pageAddr);
+ }
+ finally {
+ if (curAddr != 0L)
+ writeUnlock(curId, curPage, curAddr, write);
+
+ writeUnlock(pageId, page, pageAddr, write);
+ }
+ }
+ finally {
+ if (curPage != 0L)
+ releasePage(curId, curPage);
+ }
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page absolute pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param nextId Next page ID.
+ * @param nextPage Next page absolute pointer.
+ * @param nextAddr Next page address.
+ * @param bucket Bucket index.
+ * @return Page to recycle.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long doMerge(
+ long pageId,
+ long page,
+ long pageAddr,
+ PagesListNodeIO io,
+ long nextId,
+ long nextPage,
+ long nextAddr,
+ int bucket
+ ) throws IgniteCheckedException {
+ long prevId = io.getPreviousId(pageAddr);
+
+ if (nextId == 0L)
+ return mergeNoNext(pageId, page, pageAddr, prevId, bucket);
+ else {
+ // No one must be able to merge it while we keep a reference.
+ assert getPageId(nextAddr) == nextId;
+
+ if (prevId == 0L) { // No previous page: we are at head.
+ // These references must be updated at the same time in write locks.
+ assert PagesListNodeIO.VERSIONS.forPage(nextAddr).getPreviousId(nextAddr) == pageId;
+
+ PagesListNodeIO nextIO = PagesListNodeIO.VERSIONS.forPage(nextAddr);
+ nextIO.setPreviousId(nextAddr, 0);
+
+ if (needWalDeltaRecord(nextId, nextPage, null))
+ wal.log(new PagesListSetPreviousRecord(cacheId, nextId, 0L));
+ }
+ else // Do a fair merge: link previous and next to each other.
+ fairMerge(prevId, pageId, nextId, nextPage, nextAddr);
+
+ return recyclePage(pageId, page, pageAddr, null);
+ }
+ }
+
+ /**
+ * Link previous and next to each other.
+ * @param prevId Previous Previous page ID.
+ * @param pageId Page ID.
+ * @param nextId Next page ID.
+ * @param nextPage Next page absolute pointer.
+ * @param nextAddr Next page address.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void fairMerge(
+ final long prevId,
+ long pageId,
+ long nextId,
+ long nextPage,
+ long nextAddr)
+ throws IgniteCheckedException {
+ long prevPage = acquirePage(prevId);
+
+ try {
+ final long prevAddr = writeLock(prevId, prevPage); // No check, we keep a reference.
+ assert prevAddr != 0L;
+ try {
+ PagesListNodeIO prevIO = PagesListNodeIO.VERSIONS.forPage(prevAddr);
+ PagesListNodeIO nextIO = PagesListNodeIO.VERSIONS.forPage(nextAddr);
+
+ // These references must be updated at the same time in write locks.
+ assert prevIO.getNextId(prevAddr) == pageId;
+ assert nextIO.getPreviousId(nextAddr) == pageId;
+
+ prevIO.setNextId(prevAddr, nextId);
+
+ if (needWalDeltaRecord(prevId, prevPage, null))
+ wal.log(new PagesListSetNextRecord(cacheId, prevId, nextId));
+
+ nextIO.setPreviousId(nextAddr, prevId);
+
+ if (needWalDeltaRecord(nextId, nextPage, null))
+ wal.log(new PagesListSetPreviousRecord(cacheId, nextId, prevId));
+ }
+ finally {
+ writeUnlock(prevId, prevPage, prevAddr, true);
+ }
+ }
+ finally {
+ releasePage(prevId, prevPage);
+ }
+ }
+
+ /**
+ * Increments bucket size and updates changed flag.
+ *
+ * @param bucket Bucket number.
+ */
+ private void incrementBucketSize(int bucket) {
+ bucketsSize[bucket].incrementAndGet();
+
+ // Ok to have a race here, see the field javadoc.
+ if (!changed)
+ changed = true;
+ }
+
+ /**
+ * Increments bucket size and updates changed flag.
+ *
+ * @param bucket Bucket number.
+ */
+ private void decrementBucketSize(int bucket) {
+ bucketsSize[bucket].decrementAndGet();
+
+ // Ok to have a race here, see the field javadoc.
+ if (!changed)
+ changed = true;
+ }
+
+ /**
+ * Singleton reuse bag.
+ */
+ private static final class SingletonReuseBag implements ReuseBag {
+ /** */
+ long pageId;
+
+ /**
+ * @param pageId Page ID.
+ */
+ SingletonReuseBag(long pageId) {
+ this.pageId = pageId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addFreePage(long pageId) {
+ throw new IllegalStateException("Should never be called.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public long pollFreePage() {
+ long res = pageId;
+
+ pageId = 0L;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(SingletonReuseBag.class, this, "pageId", U.hexLong(pageId));
+ }
+ }
+
+ /**
+ *
+ */
+ public static final class Stripe {
+ /** */
+ public volatile long tailId;
+
+ /** */
+ volatile boolean empty;
+
+ /**
+ * @param tailId Tail ID.
+ * @param empty Empty flag.
+ */
+ Stripe(long tailId, boolean empty) {
+ this.tailId = tailId;
+ this.empty = empty;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java
new file mode 100644
index 0000000..41e1bb5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListMetaIO.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist.io;
+
+import java.util.Map;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.util.GridLongList;
+
+/**
+ *
+ */
+public class PagesListMetaIO extends PageIO {
+ /** */
+ private static final int CNT_OFF = COMMON_HEADER_END;
+
+ /** */
+ private static final int NEXT_META_PAGE_OFF = CNT_OFF + 2;
+
+ /** */
+ private static final int ITEMS_OFF = NEXT_META_PAGE_OFF + 8;
+
+ /** */
+ private static final int ITEM_SIZE = 10;
+
+ /** */
+ public static final IOVersions<PagesListMetaIO> VERSIONS = new IOVersions<>(
+ new PagesListMetaIO(1)
+ );
+
+ /**
+ * @param ver Page format version.
+ */
+ private PagesListMetaIO(int ver) {
+ super(T_PAGE_LIST_META, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setCount(pageAddr, 0);
+ setNextMetaPageId(pageAddr, 0L);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Stored items count.
+ */
+ private int getCount(long pageAddr) {
+ return PageUtils.getShort(pageAddr, CNT_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Stored items count.
+ */
+ private void setCount(long pageAddr, int cnt) {
+ assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
+
+ PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Next meta page ID.
+ */
+ public long getNextMetaPageId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, NEXT_META_PAGE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param metaPageId Next meta page ID.
+ */
+ public void setNextMetaPageId(long pageAddr, long metaPageId) {
+ PageUtils.putLong(pageAddr, NEXT_META_PAGE_OFF, metaPageId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public void resetCount(long pageAddr) {
+ setCount(pageAddr, 0);
+ }
+
+ /**
+ * @param pageSize Page size.
+ * @param pageAddr Page address.
+ * @param bucket Bucket number.
+ * @param tails Tails.
+ * @param tailsOff Tails offset.
+ * @return Number of items written.
+ */
+ public int addTails(int pageSize, long pageAddr, int bucket, PagesList.Stripe[] tails, int tailsOff) {
+ assert bucket >= 0 && bucket <= Short.MAX_VALUE : bucket;
+
+ int cnt = getCount(pageAddr);
+ int cap = getCapacity(pageSize, pageAddr);
+
+ if (cnt == cap)
+ return 0;
+
+ int off = offset(cnt);
+
+ int write = Math.min(cap - cnt, tails.length - tailsOff);
+
+ for (int i = 0; i < write; i++) {
+ PageUtils.putShort(pageAddr, off, (short)bucket);
+ PageUtils.putLong(pageAddr, off + 2, tails[tailsOff].tailId);
+
+ tailsOff++;
+
+ off += ITEM_SIZE;
+ }
+
+ setCount(pageAddr, cnt + write);
+
+ return write;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param res Results map.
+ */
+ public void getBucketsData(long pageAddr, Map<Integer, GridLongList> res) {
+ int cnt = getCount(pageAddr);
+
+ assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
+
+ if (cnt == 0)
+ return;
+
+ int off = offset(0);
+
+ for (int i = 0; i < cnt; i++) {
+ Integer bucket = (int)PageUtils.getShort(pageAddr, off);
+ assert bucket >= 0 && bucket <= Short.MAX_VALUE : bucket;
+
+ long tailId = PageUtils.getLong(pageAddr, off + 2);
+ assert tailId != 0;
+
+ GridLongList list = res.get(bucket);
+
+ if (list == null)
+ res.put(bucket, list = new GridLongList());
+
+ list.add(tailId);
+
+ off += ITEM_SIZE;
+ }
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Maximum number of items which can be stored in buffer.
+ */
+ private int getCapacity(int pageSize, long pageAddr) {
+ return (pageSize - ITEMS_OFF) / ITEM_SIZE;
+ }
+
+ /**
+ * @param idx Item index.
+ * @return Item offset.
+ */
+ private int offset(int idx) {
+ return ITEMS_OFF + ITEM_SIZE * idx;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListNodeIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListNodeIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListNodeIO.java
new file mode 100644
index 0000000..7db89eb
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/io/PagesListNodeIO.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist.io;
+
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+
+import static org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler.copyMemory;
+
+/**
+ * TODO optimize: now we have slow {@link #removePage(long, long)}
+ */
+public class PagesListNodeIO extends PageIO {
+ /** */
+ public static final IOVersions<PagesListNodeIO> VERSIONS = new IOVersions<>(
+ new PagesListNodeIO(1)
+ );
+
+ /** */
+ private static final int PREV_PAGE_ID_OFF = COMMON_HEADER_END;
+
+ /** */
+ private static final int NEXT_PAGE_ID_OFF = PREV_PAGE_ID_OFF + 8;
+
+ /** */
+ private static final int CNT_OFF = NEXT_PAGE_ID_OFF + 8;
+
+ /** */
+ private static final int PAGE_IDS_OFF = CNT_OFF + 2;
+
+ /**
+ * @param ver Page format version.
+ */
+ protected PagesListNodeIO(int ver) {
+ super(T_PAGE_LIST_NODE, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setEmpty(pageAddr);
+
+ setPreviousId(pageAddr, 0L);
+ setNextId(pageAddr, 0L);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ private void setEmpty(long pageAddr) {
+ setCount(pageAddr, 0);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Next page ID.
+ */
+ public long getNextId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, NEXT_PAGE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param nextId Next page ID.
+ */
+ public void setNextId(long pageAddr, long nextId) {
+ PageUtils.putLong(pageAddr, NEXT_PAGE_ID_OFF, nextId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Previous page ID.
+ */
+ public long getPreviousId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, PREV_PAGE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param prevId Previous page ID.
+ */
+ public void setPreviousId(long pageAddr, long prevId) {
+ PageUtils.putLong(pageAddr, PREV_PAGE_ID_OFF, prevId);
+ }
+
+ /**
+ * Gets total count of entries in this page. Does not change the buffer state.
+ *
+ * @param pageAddr Page address to get count from.
+ * @return Total number of entries.
+ */
+ public int getCount(long pageAddr) {
+ return PageUtils.getShort(pageAddr, CNT_OFF);
+ }
+
+ /**
+ * Sets total count of entries in this page. Does not change the buffer state.
+ *
+ * @param pageAddr Page address to write to.
+ * @param cnt Count.
+ */
+ private void setCount(long pageAddr, int cnt) {
+ assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
+
+ PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
+ }
+
+ /**
+ * Gets capacity of this page in items.
+ *
+ * @param pageSize Page size.
+ * @return Capacity of this page in items.
+ */
+ private int getCapacity(int pageSize) {
+ return (pageSize - PAGE_IDS_OFF) >>> 3; // /8
+ }
+
+ /**
+ * @param idx Item index.
+ * @return Item offset.
+ */
+ private int offset(int idx) {
+ return PAGE_IDS_OFF + 8 * idx;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Item index.
+ * @return Item at the given index.
+ */
+ private long getAt(long pageAddr, int idx) {
+ return PageUtils.getLong(pageAddr, offset(idx));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Item index.
+ * @param pageId Item value to write.
+ */
+ private void setAt(long pageAddr, int idx, long pageId) {
+ PageUtils.putLong(pageAddr, offset(idx), pageId);
+ }
+
+ /**
+ * Adds page to the end of pages list.
+ *
+ * @param pageAddr Page address.
+ * @param pageId Page ID.
+ * @param pageSize Page size.
+ * @return Total number of items in this page.
+ */
+ public int addPage(long pageAddr, long pageId, int pageSize) {
+ int cnt = getCount(pageAddr);
+
+ if (cnt == getCapacity(pageSize))
+ return -1;
+
+ setAt(pageAddr, cnt, pageId);
+ setCount(pageAddr, cnt + 1);
+
+ return cnt;
+ }
+
+ /**
+ * Removes any page from the pages list.
+ *
+ * @param pageAddr Page address.
+ * @return Removed page ID.
+ */
+ public long takeAnyPage(long pageAddr) {
+ int cnt = getCount(pageAddr);
+
+ if (cnt == 0)
+ return 0L;
+
+ setCount(pageAddr, --cnt);
+
+ return getAt(pageAddr, cnt);
+ }
+
+ /**
+ * Removes the given page ID from the pages list.
+ *
+ * @param pageAddr Page address.
+ * @param dataPageId Page ID to remove.
+ * @return {@code true} if page was in the list and was removed, {@code false} otherwise.
+ */
+ public boolean removePage(long pageAddr, long dataPageId) {
+ assert dataPageId != 0;
+
+ int cnt = getCount(pageAddr);
+
+ for (int i = 0; i < cnt; i++) {
+ if (getAt(pageAddr, i) == dataPageId) {
+ if (i != cnt - 1)
+ copyMemory(pageAddr, pageAddr, offset(i + 1), offset(i), 8 * (cnt - i - 1));
+
+ setCount(pageAddr, cnt - 1);
+
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return {@code True} if there are no items in this page.
+ */
+ public boolean isEmpty(long pageAddr) {
+ return getCount(pageAddr) == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
new file mode 100644
index 0000000..e533f41
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/CheckpointMetricsTracker.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+
+/**
+ * Tracks various checkpoint phases and stats.
+ *
+ * Assumed sequence of events:
+ * <ol>
+ * <li>Checkpoint start</li>
+ * <li>CP Lock wait start</li>
+ * <li>CP mark start</li>
+ * <li>CP Lock release</li>
+ * <li>Pages write start</li>
+ * <li>fsync start</li>
+ * <li>Checkpoint end</li>
+ * </ol>
+ */
+public class CheckpointMetricsTracker {
+ /** */
+ private static final AtomicIntegerFieldUpdater<CheckpointMetricsTracker> DATA_PAGES_UPDATER =
+ AtomicIntegerFieldUpdater.newUpdater(CheckpointMetricsTracker.class, "dataPages");
+
+ /** */
+ private static final AtomicIntegerFieldUpdater<CheckpointMetricsTracker> COW_PAGES_UPDATER =
+ AtomicIntegerFieldUpdater.newUpdater(CheckpointMetricsTracker.class, "cowPages");
+
+ /** */
+ private volatile int dataPages;
+
+ /** */
+ private volatile int cowPages;
+
+ /** */
+ private long cpStart = System.currentTimeMillis();
+
+ /** */
+ private long cpLockWaitStart;
+
+ /** */
+ private long cpMarkStart;
+
+ /** */
+ private long cpLockRelease;
+
+ /** */
+ private long cpPagesWriteStart;
+
+ /** */
+ private long cpFsyncStart;
+
+ /** */
+ private long cpEnd;
+
+ /**
+ *
+ */
+ public void onCowPageWritten() {
+ COW_PAGES_UPDATER.incrementAndGet(this);
+ }
+
+ /**
+ *
+ */
+ public void onDataPageWritten() {
+ DATA_PAGES_UPDATER.incrementAndGet(this);
+ }
+
+ /**
+ * @return COW pages.
+ */
+ public int cowPagesWritten() {
+ return cowPages;
+ }
+
+ /**
+ * @return Data pages written.
+ */
+ public int dataPagesWritten() {
+ return dataPages;
+ }
+
+ /**
+ *
+ */
+ public void onLockWaitStart() {
+ cpLockWaitStart = System.currentTimeMillis();
+ }
+
+ /**
+ *
+ */
+ public void onMarkStart() {
+ cpMarkStart = System.currentTimeMillis();
+ }
+
+ /**
+ *
+ */
+ public void onLockRelease() {
+ cpLockRelease = System.currentTimeMillis();
+ }
+
+ /**
+ *
+ */
+ public void onPagesWriteStart() {
+ cpPagesWriteStart = System.currentTimeMillis();
+ }
+
+ /**
+ *
+ */
+ public void onFsyncStart() {
+ cpFsyncStart = System.currentTimeMillis();
+ }
+
+ /**
+ *
+ */
+ public void onEnd() {
+ cpEnd = System.currentTimeMillis();
+ }
+
+ /**
+ * @return Total checkpoint duration.
+ */
+ public long totalDuration() {
+ return cpEnd - cpStart;
+ }
+
+ /**
+ * @return Checkpoint lock wait duration.
+ */
+ public long lockWaitDuration() {
+ return cpMarkStart - cpLockWaitStart;
+ }
+
+ /**
+ * @return Checkpoint mark duration.
+ */
+ public long markDuration() {
+ return cpPagesWriteStart - cpMarkStart;
+ }
+
+ /**
+ * @return Checkpoint lock hold duration.
+ */
+ public long lockHoldDuration() {
+ return cpLockRelease - cpMarkStart;
+ }
+
+ /**
+ * @return Pages write duration.
+ */
+ public long pagesWriteDuration() {
+ return cpFsyncStart - cpPagesWriteStart;
+ }
+
+ /**
+ * @return Checkpoint fsync duration.
+ */
+ public long fsyncDuration() {
+ return cpEnd - cpFsyncStart;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
new file mode 100644
index 0000000..eb172c9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/EvictCandidate.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class EvictCandidate {
+ /** */
+ private int tag;
+
+ /** */
+ @GridToStringExclude
+ private long relPtr;
+
+ /** */
+ @GridToStringInclude
+ private FullPageId fullId;
+
+ /**
+ * @param tag Tag.
+ * @param relPtr Relative pointer.
+ * @param fullId Full page ID.
+ */
+ public EvictCandidate(int tag, long relPtr, FullPageId fullId) {
+ this.tag = tag;
+ this.relPtr = relPtr;
+ this.fullId = fullId;
+ }
+
+ /**
+ * @return Tag.
+ */
+ public int tag() {
+ return tag;
+ }
+
+ /**
+ * @return Relative pointer.
+ */
+ public long relativePointer() {
+ return relPtr;
+ }
+
+ /**
+ * @return Index.
+ */
+ public FullPageId fullId() {
+ return fullId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(EvictCandidate.class, this, "relPtr", U.hexLong(relPtr));
+ }
+}
[38/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
new file mode 100644
index 0000000..eb9f4df
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -0,0 +1,1316 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl;
+import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionCountersIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.lang.GridCursor;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Used when persistence enabled.
+ */
+public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl implements DbCheckpointListener {
+ /** */
+ private MetaStore metaStore;
+
+ /** */
+ private ReuseListImpl reuseList;
+
+ /** {@inheritDoc} */
+ @Override protected void initDataStructures() throws IgniteCheckedException {
+ Metas metas = getOrAllocateCacheMetas();
+
+ RootPage reuseListRoot = metas.reuseListRoot;
+
+ reuseList = new ReuseListImpl(grp.groupId(),
+ grp.cacheOrGroupName(),
+ grp.memoryPolicy().pageMemory(),
+ ctx.wal(),
+ reuseListRoot.pageId().pageId(),
+ reuseListRoot.isAllocated());
+
+ RootPage metastoreRoot = metas.treeRoot;
+
+ metaStore = new MetadataStorage(grp.memoryPolicy().pageMemory(),
+ ctx.wal(),
+ globalRemoveId(),
+ grp.groupId(),
+ PageIdAllocator.INDEX_PARTITION,
+ PageIdAllocator.FLAG_IDX,
+ reuseList,
+ metastoreRoot.pageId().pageId(),
+ metastoreRoot.isAllocated());
+
+ ((GridCacheDatabaseSharedManager)ctx.database()).addCheckpointListener(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCacheStarted(GridCacheContext cctx) throws IgniteCheckedException {
+ if (cctx.affinityNode() && cctx.ttl().eagerTtlEnabled() && pendingEntries == null) {
+ ctx.database().checkpointReadLock();
+
+ try {
+ final String name = "PendingEntries";
+
+ RootPage pendingRootPage = metaStore.getOrAllocateForTree(name);
+
+ pendingEntries = new PendingEntriesTree(
+ grp,
+ name,
+ grp.memoryPolicy().pageMemory(),
+ pendingRootPage.pageId().pageId(),
+ reuseList,
+ pendingRootPage.isAllocated()
+ );
+ }
+ finally {
+ ctx.database().checkpointReadUnlock();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected CacheDataStore createCacheDataStore0(final int p)
+ throws IgniteCheckedException {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ctx.database();
+
+ boolean exists = ctx.pageStore() != null
+ && ctx.pageStore().exists(grp.groupId(), p);
+
+ return new GridCacheDataStore(p, exists);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
+ assert grp.memoryPolicy().pageMemory() instanceof PageMemoryEx;
+
+ reuseList.saveMetadata();
+
+ boolean metaWasUpdated = false;
+
+ for (CacheDataStore store : partDataStores.values()) {
+ RowStore rowStore = store.rowStore();
+
+ if (rowStore == null)
+ continue;
+
+ metaWasUpdated |= saveStoreMetadata(store, ctx, !metaWasUpdated, false);
+ }
+ }
+
+ /**
+ * @param store Store to save metadata.
+ * @throws IgniteCheckedException If failed.
+ */
+ private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean saveMeta,
+ boolean beforeDestroy) throws IgniteCheckedException {
+ RowStore rowStore0 = store.rowStore();
+
+ boolean beforeSnapshot = ctx != null && ctx.nextSnapshot();
+
+ boolean wasSaveToMeta = false;
+
+ if (rowStore0 != null) {
+ FreeListImpl freeList = (FreeListImpl)rowStore0.freeList();
+
+ freeList.saveMetadata();
+
+ long updCntr = store.updateCounter();
+ int size = store.fullSize();
+ long rmvId = globalRemoveId().get();
+
+ PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+ IgniteWriteAheadLogManager wal = this.ctx.wal();
+
+ if (size > 0 || updCntr > 0) {
+ int state = -1;
+
+ if (beforeDestroy)
+ state = GridDhtPartitionState.EVICTED.ordinal();
+ else {
+ // localPartition will not acquire writeLock here because create=false.
+ GridDhtLocalPartition part = grp.topology().localPartition(store.partId(),
+ AffinityTopologyVersion.NONE, false);
+
+ if (part != null && part.state() != GridDhtPartitionState.EVICTED)
+ state = part.state().ordinal();
+ }
+
+ // Do not save meta for evicted partitions on next checkpoints.
+ if (state == -1)
+ return false;
+
+ int grpId = grp.groupId();
+ long partMetaId = pageMem.partitionMetaPageId(grpId, store.partId());
+ long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+
+ try {
+ long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
+
+ if (pageAddr == 0L) {
+ U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage +
+ ", saveMeta=" + saveMeta + ", beforeDestroy=" + beforeDestroy + ", size=" + size +
+ ", updCntr=" + updCntr + ", state=" + state + ']');
+
+ return false;
+ }
+
+ boolean changed = false;
+
+ try {
+ PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+
+ changed |= io.setUpdateCounter(pageAddr, updCntr);
+ changed |= io.setGlobalRemoveId(pageAddr, rmvId);
+ changed |= io.setSize(pageAddr, size);
+
+ changed |= io.setPartitionState(pageAddr, (byte)state);
+
+ long cntrsPageId;
+
+ if (grp.sharedGroup()) {
+ cntrsPageId = io.getCountersPageId(pageAddr);
+
+ byte[] data = serializeCacheSizes(store.cacheSizes());
+
+ int items = data.length / 12;
+ int written = 0;
+ int pageSize = pageMem.pageSize();
+
+ boolean init = cntrsPageId == 0;
+
+ if (init) {
+ cntrsPageId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
+
+ io.setCountersPageId(pageAddr, cntrsPageId);
+
+ changed = true;
+ }
+
+ long nextId = cntrsPageId;
+
+ while (written != items) {
+ final long curId = nextId;
+ final long curPage = pageMem.acquirePage(grpId, curId);
+
+ try {
+ final long curAddr = pageMem.writeLock(grpId, curId, curPage);
+
+ assert curAddr != 0;
+
+ try {
+ PagePartitionCountersIO partMetaIo;
+
+ if (init) {
+ partMetaIo = PagePartitionCountersIO.VERSIONS.latest();
+
+ partMetaIo.initNewPage(curAddr, curId, pageSize);
+ }
+ else
+ partMetaIo = PageIO.getPageIO(curAddr);
+
+ written += partMetaIo.writeCacheSizes(pageSize, curAddr, data, written);
+
+ nextId = partMetaIo.getNextCountersPageId(curAddr);
+
+ if(written != items && (init = nextId == 0)) {
+ //allocate new counters page
+ nextId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
+ partMetaIo.setNextCountersPageId(curAddr, nextId);
+ }
+ }
+ finally {
+ // Write full page
+ pageMem.writeUnlock(grpId, curId, curPage, Boolean.TRUE, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, curId, curPage);
+ }
+ }
+ }
+ else
+ cntrsPageId = 0L;
+
+ int pageCnt;
+
+ if (beforeSnapshot) {
+ pageCnt = this.ctx.pageStore().pages(grpId, store.partId());
+ io.setCandidatePageCount(pageAddr, pageCnt);
+
+ if (saveMeta) {
+ long metaPageId = pageMem.metaPageId(grpId);
+ long metaPage = pageMem.acquirePage(grpId, metaPageId);
+
+ try {
+ long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
+
+ try {
+ long nextSnapshotTag = io.getNextSnapshotTag(metaPageAddr);
+
+ io.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1);
+
+ if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId,
+ metaPage, wal, null))
+ wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId,
+ nextSnapshotTag + 1));
+
+ addPartition(ctx.partitionStatMap(), metaPageAddr, io, grpId, PageIdAllocator.INDEX_PARTITION,
+ this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, metaPageId, metaPage);
+ }
+
+ wasSaveToMeta = true;
+ }
+
+ GridDhtPartitionMap partMap = grp.topology().localPartitionMap();
+
+ if (partMap.containsKey(store.partId()) &&
+ partMap.get(store.partId()) == GridDhtPartitionState.OWNING)
+ addPartition(ctx.partitionStatMap(), pageAddr, io, grpId, store.partId(),
+ this.ctx.pageStore().pages(grpId, store.partId()));
+
+ changed = true;
+ }
+ else
+ pageCnt = io.getCandidatePageCount(pageAddr);
+
+ if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
+ wal.log(new MetaPageUpdatePartitionDataRecord(
+ grpId,
+ partMetaId,
+ updCntr,
+ rmvId,
+ size,
+ cntrsPageId,
+ (byte)state,
+ pageCnt
+ ));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, partMetaId, partMetaPage);
+ }
+ }
+ }
+
+ return wasSaveToMeta;
+ }
+
+ /**
+ * @param cacheSizes Cache sizes.
+ * @return Serialized cache sizes
+ */
+ private byte[] serializeCacheSizes(Map<Integer, Long> cacheSizes) {
+ // Item size = 4 bytes (cache ID) + 8 bytes (cache size) = 12 bytes
+ byte[] data = new byte[cacheSizes.size() * 12];
+ long off = GridUnsafe.BYTE_ARR_OFF;
+
+ for (Map.Entry<Integer, Long> entry : cacheSizes.entrySet()) {
+ GridUnsafe.putInt(data, off, entry.getKey()); off += 4;
+ GridUnsafe.putLong(data, off, entry.getValue()); off += 8;
+ }
+
+ return data;
+ }
+
+ /**
+ * @param map Map to add values to.
+ * @param pageAddr page address
+ * @param io Page Meta IO
+ * @param cacheId Cache ID.
+ * @param partition Partition ID.
+ * @param pages Number of pages to add.
+ */
+ private static void addPartition(
+ Map<T2<Integer, Integer>, T2<Integer, Integer>> map,
+ long pageAddr,
+ PagePartitionMetaIO io,
+ int cacheId,
+ int partition,
+ int pages
+ ) {
+ if (pages <= 1)
+ return;
+
+ assert PageIO.getPageId(pageAddr) != 0;
+
+ int lastAllocatedIdx = io.getLastPageCount(pageAddr);
+ map.put(new T2<>(cacheId, partition), new T2<>(lastAllocatedIdx, pages));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void destroyCacheDataStore0(CacheDataStore store) throws IgniteCheckedException {
+ ctx.database().checkpointReadLock();
+
+ try {
+ int p = store.partId();
+
+ saveStoreMetadata(store, null, false, true);
+
+ PageMemoryEx pageMemory = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+
+ int tag = pageMemory.invalidate(grp.groupId(), p);
+
+ ctx.wal().log(new PartitionDestroyRecord(grp.groupId(), p));
+
+ ctx.pageStore().onPartitionDestroyed(grp.groupId(), p, tag);
+ }
+ finally {
+ ctx.database().checkpointReadUnlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onPartitionCounterUpdated(int part, long cntr) {
+ CacheDataStore store = partDataStores.get(part);
+
+ assert store != null;
+
+ long oldCnt = store.updateCounter();
+
+ if (oldCnt < cntr)
+ store.updateCounter(cntr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onPartitionInitialCounterUpdated(int part, long cntr) {
+ CacheDataStore store = partDataStores.get(part);
+
+ assert store != null;
+
+ long oldCnt = store.initialUpdateCounter();
+
+ if (oldCnt < cntr)
+ store.updateInitialCounter(cntr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long lastUpdatedPartitionCounter(int part) {
+ return partDataStores.get(part).updateCounter();
+ }
+
+ /** {@inheritDoc} */
+ @Override public RootPage rootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException {
+ if (grp.sharedGroup())
+ idxName = Integer.toString(cacheId) + "_" + idxName;
+
+ return metaStore.getOrAllocateForTree(idxName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dropRootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException {
+ if (grp.sharedGroup())
+ idxName = Integer.toString(cacheId) + "_" + idxName;
+
+ metaStore.dropRootPage(idxName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public ReuseList reuseListForIndex(String idxName) {
+ return reuseList;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() {
+ if (grp.affinityNode())
+ ((GridCacheDatabaseSharedManager)ctx.database()).removeCheckpointListener(this);
+ }
+
+ /**
+ * @return Meta root pages info.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
+ PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+ IgniteWriteAheadLogManager wal = ctx.wal();
+
+ int grpId = grp.groupId();
+ long metaId = pageMem.metaPageId(grpId);
+ long metaPage = pageMem.acquirePage(grpId, metaId);
+
+ try {
+ final long pageAddr = pageMem.writeLock(grpId, metaId, metaPage);
+
+ boolean allocated = false;
+
+ try {
+ long metastoreRoot, reuseListRoot;
+
+ if (PageIO.getType(pageAddr) != PageIO.T_META) {
+ PageMetaIO pageIO = PageMetaIO.VERSIONS.latest();
+
+ pageIO.initNewPage(pageAddr, metaId, pageMem.pageSize());
+
+ metastoreRoot = pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX);
+ reuseListRoot = pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX);
+
+ pageIO.setTreeRoot(pageAddr, metastoreRoot);
+ pageIO.setReuseListRoot(pageAddr, reuseListRoot);
+
+ if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaId, metaPage, wal, null))
+ wal.log(new MetaPageInitRecord(
+ grpId,
+ metaId,
+ pageIO.getType(),
+ pageIO.getVersion(),
+ metastoreRoot,
+ reuseListRoot
+ ));
+
+ allocated = true;
+ }
+ else {
+ PageMetaIO pageIO = PageIO.getPageIO(pageAddr);
+
+ metastoreRoot = pageIO.getTreeRoot(pageAddr);
+ reuseListRoot = pageIO.getReuseListRoot(pageAddr);
+
+ assert reuseListRoot != 0L;
+ }
+
+ return new Metas(
+ new RootPage(new FullPageId(metastoreRoot, grpId), allocated),
+ new RootPage(new FullPageId(reuseListRoot, grpId), allocated));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, metaId, metaPage, null, allocated);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, metaId, metaPage);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteRebalanceIterator rebalanceIterator(int part, AffinityTopologyVersion topVer,
+ Long partCntrSince) throws IgniteCheckedException {
+ if (partCntrSince == null)
+ return super.rebalanceIterator(part, topVer, partCntrSince);
+
+ GridCacheDatabaseSharedManager database = (GridCacheDatabaseSharedManager)ctx.database();
+
+ try {
+ WALPointer startPtr = database.searchPartitionCounter(grp.groupId(), part, partCntrSince);
+
+ if (startPtr == null) {
+ assert false : "partCntr=" + partCntrSince + ", reservations=" + S.toString(Map.class, database.reservedForPreloading());
+
+ return super.rebalanceIterator(part, topVer, partCntrSince);
+ }
+
+ WALIterator it = ctx.wal().replay(startPtr);
+
+ return new RebalanceIteratorAdapter(grp, it, part);
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Failed to create WAL-based rebalance iterator (a full partition will transferred to a " +
+ "remote node) [part=" + part + ", partCntrSince=" + partCntrSince + ", err=" + e + ']');
+
+ return super.rebalanceIterator(part, topVer, partCntrSince);
+ }
+ }
+
+ /**
+ *
+ */
+ private static class RebalanceIteratorAdapter implements IgniteRebalanceIterator {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Cache group caches. */
+ private final Set<Integer> cacheGrpCaches;
+
+ /** WAL iterator. */
+ private final WALIterator walIt;
+
+ /** Partition to scan. */
+ private final int part;
+
+ /** */
+ private Iterator<DataEntry> entryIt;
+
+ /** */
+ private CacheDataRow next;
+
+ /**
+ * @param grp Cache group.
+ * @param walIt WAL iterator.
+ * @param part Partition ID.
+ */
+ private RebalanceIteratorAdapter(CacheGroupContext grp, WALIterator walIt, int part) {
+ this.cacheGrpCaches = grp.cacheIds();
+ this.walIt = walIt;
+ this.part = part;
+
+ advance();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean historical() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ walIt.close();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isClosed() {
+ return walIt.isClosed();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasNextX() {
+ return hasNext();
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheDataRow nextX() throws IgniteCheckedException {
+ return next();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeX() throws IgniteCheckedException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Iterator<CacheDataRow> iterator() {
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasNext() {
+ return next != null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheDataRow next() {
+ if (next == null)
+ throw new NoSuchElementException();
+
+ CacheDataRow val = next;
+
+ advance();
+
+ return val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ *
+ */
+ private void advance() {
+ next = null;
+
+ while (true) {
+ if (entryIt != null) {
+ while (entryIt.hasNext()) {
+ DataEntry entry = entryIt.next();
+
+ if (entry.partitionId() == part && cacheGrpCaches.contains(entry.cacheId())) {
+
+ next = new DataEntryRow(entry);
+
+ return;
+ }
+ }
+ }
+
+ entryIt = null;
+
+ while (walIt.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> rec = walIt.next();
+
+ if (rec.get2() instanceof DataRecord) {
+ DataRecord data = (DataRecord)rec.get2();
+
+ entryIt = data.writeEntries().iterator();
+ // Move on to the next valid data entry.
+
+ break;
+ }
+ }
+
+ if (entryIt == null)
+ return;
+ }
+ }
+ }
+
+ /**
+ * Data entry row.
+ */
+ private static class DataEntryRow implements CacheDataRow {
+ /** */
+ private final DataEntry entry;
+
+ /**
+ * @param entry Data entry.
+ */
+ private DataEntryRow(DataEntry entry) {
+ this.entry = entry;
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyCacheObject key() {
+ return entry.key();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void key(KeyCacheObject key) {
+ throw new IllegalStateException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheObject value() {
+ return entry.value();
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCacheVersion version() {
+ return entry.writeVersion();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long expireTime() {
+ return entry.expireTime();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return entry.partitionId();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long link() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void link(long link) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hash() {
+ return entry.key().hashCode();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int cacheId() {
+ return entry.cacheId();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class Metas {
+ /** */
+ @GridToStringInclude
+ private final RootPage reuseListRoot;
+
+ /** */
+ @GridToStringInclude
+ private final RootPage treeRoot;
+
+ /**
+ * @param treeRoot Metadata storage root.
+ * @param reuseListRoot Reuse list root.
+ */
+ Metas(RootPage treeRoot, RootPage reuseListRoot) {
+ this.treeRoot = treeRoot;
+ this.reuseListRoot = reuseListRoot;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(Metas.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private class GridCacheDataStore implements CacheDataStore {
+ /** */
+ private final int partId;
+
+ /** */
+ private String name;
+
+ /** */
+ private volatile FreeListImpl freeList;
+
+ /** */
+ private volatile CacheDataStore delegate;
+
+ /** */
+ private final boolean exists;
+
+ /** */
+ private final AtomicBoolean init = new AtomicBoolean();
+
+ /** */
+ private final CountDownLatch latch = new CountDownLatch(1);
+
+ /**
+ * @param partId Partition.
+ * @param exists {@code True} if store for this index exists.
+ */
+ private GridCacheDataStore(int partId, boolean exists) {
+ this.partId = partId;
+ this.exists = exists;
+
+ name = treeName(partId);
+ }
+
+ /**
+ * @return Store delegate.
+ * @throws IgniteCheckedException If failed.
+ */
+ private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException {
+ CacheDataStore delegate0 = delegate;
+
+ if (delegate0 != null)
+ return delegate0;
+
+ if (checkExists) {
+ if (!exists)
+ return null;
+ }
+
+ IgniteCacheDatabaseSharedManager dbMgr = ctx.database();
+
+ dbMgr.checkpointReadLock();
+
+ if (init.compareAndSet(false, true)) {
+ try {
+ Metas metas = getOrAllocatePartitionMetas();
+
+ RootPage reuseRoot = metas.reuseListRoot;
+
+ freeList = new FreeListImpl(
+ grp.groupId(),
+ grp.cacheOrGroupName() + "-" + partId,
+ grp.memoryPolicy().memoryMetrics(),
+ grp.memoryPolicy(),
+ null,
+ ctx.wal(),
+ reuseRoot.pageId().pageId(),
+ reuseRoot.isAllocated()) {
+ @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+ return pageMem.allocatePage(cacheId, partId, PageIdAllocator.FLAG_DATA);
+ }
+ };
+
+ CacheDataRowStore rowStore = new CacheDataRowStore(grp, freeList, partId);
+
+ RootPage treeRoot = metas.treeRoot;
+
+ CacheDataTree dataTree = new CacheDataTree(
+ grp,
+ name,
+ freeList,
+ rowStore,
+ treeRoot.pageId().pageId(),
+ treeRoot.isAllocated()) {
+ @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+ return pageMem.allocatePage(cacheId, partId, PageIdAllocator.FLAG_DATA);
+ }
+ };
+
+ PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+
+ delegate0 = new CacheDataStoreImpl(partId, name, rowStore, dataTree);
+
+ int grpId = grp.groupId();
+ long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
+ long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+
+ try {
+ long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
+
+ try {
+ if (PageIO.getType(pageAddr) != 0) {
+ PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.latest();
+
+ Map<Integer, Long> cacheSizes = null;
+
+ if (grp.sharedGroup()) {
+ long cntrsPageId = io.getCountersPageId(pageAddr);
+
+ if (cntrsPageId != 0L) {
+ cacheSizes = new HashMap<>();
+
+ long nextId = cntrsPageId;
+
+ while (true){
+ final long curId = nextId;
+ final long curPage = pageMem.acquirePage(grpId, curId);
+
+ try {
+ final long curAddr = pageMem.readLock(grpId, curId, curPage);
+
+ assert curAddr != 0;
+
+ try {
+ PagePartitionCountersIO cntrsIO = PageIO.getPageIO(curAddr);
+
+ if (cntrsIO.readCacheSizes(curAddr, cacheSizes))
+ break;
+
+ nextId = cntrsIO.getNextCountersPageId(curAddr);
+
+ assert nextId != 0;
+ }
+ finally {
+ pageMem.readUnlock(grpId, curId, curPage);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, curId, curPage);
+ }
+ }
+ }
+ }
+
+ delegate0.init(io.getSize(pageAddr), io.getUpdateCounter(pageAddr), cacheSizes);
+
+ globalRemoveId().setIfGreater(io.getGlobalRemoveId(pageAddr));
+ }
+ }
+ finally {
+ pageMem.readUnlock(grpId, partMetaId, partMetaPage);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, partMetaId, partMetaPage);
+ }
+
+ delegate = delegate0;
+ }
+ finally {
+ latch.countDown();
+
+ dbMgr.checkpointReadUnlock();
+ }
+ }
+ else {
+ dbMgr.checkpointReadUnlock();
+
+ U.await(latch);
+
+ delegate0 = delegate;
+
+ if (delegate0 == null)
+ throw new IgniteCheckedException("Cache store initialization failed.");
+ }
+
+ return delegate0;
+ }
+
+ /**
+ * @return Partition metas.
+ */
+ private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException {
+ PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+ IgniteWriteAheadLogManager wal = ctx.wal();
+
+ int grpId = grp.groupId();
+ long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
+ long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+ try {
+ boolean allocated = false;
+ long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
+
+ try {
+ long treeRoot, reuseListRoot;
+
+ // Initialize new page.
+ if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
+ PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.latest();
+
+ io.initNewPage(pageAddr, partMetaId, pageMem.pageSize());
+
+ treeRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_DATA);
+ reuseListRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_DATA);
+
+ assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA;
+ assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA;
+
+ io.setTreeRoot(pageAddr, treeRoot);
+ io.setReuseListRoot(pageAddr, reuseListRoot);
+
+ if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
+ wal.log(new MetaPageInitRecord(
+ grpId,
+ partMetaId,
+ io.getType(),
+ io.getVersion(),
+ treeRoot,
+ reuseListRoot
+ ));
+
+ allocated = true;
+ }
+ else {
+ PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
+
+ treeRoot = io.getTreeRoot(pageAddr);
+ reuseListRoot = io.getReuseListRoot(pageAddr);
+
+ assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
+ U.hexLong(treeRoot) + ", part=" + partId + ", grpId=" + grpId;
+ assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
+ U.hexLong(reuseListRoot) + ", part=" + partId + ", grpId=" + grpId;
+ }
+
+ return new Metas(
+ new RootPage(new FullPageId(treeRoot, grpId), allocated),
+ new RootPage(new FullPageId(reuseListRoot, grpId), allocated));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, allocated);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, partMetaId, partMetaPage);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partId() {
+ return partId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String name() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RowStore rowStore() {
+ CacheDataStore delegate0 = delegate;
+
+ return delegate0 == null ? null : delegate0.rowStore();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int fullSize() {
+ try {
+ CacheDataStore delegate0 = init0(true);
+
+ return delegate0 == null ? 0 : delegate0.fullSize();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int cacheSize(int cacheId) {
+ try {
+ CacheDataStore delegate0 = init0(true);
+
+ return delegate0 == null ? 0 : delegate0.cacheSize(cacheId);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<Integer, Long> cacheSizes() {
+ try {
+ CacheDataStore delegate0 = init0(true);
+
+ return delegate0 == null ? null : delegate0.cacheSizes();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long updateCounter() {
+ try {
+ CacheDataStore delegate0 = init0(true);
+
+ return delegate0 == null ? 0 : delegate0.updateCounter();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(long size, long updCntr, @Nullable Map<Integer, Long> cacheSizes) {
+ throw new IllegalStateException("Should be never called.");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void updateCounter(long val) {
+ try {
+ CacheDataStore delegate0 = init0(false);
+
+ if (delegate0 != null)
+ delegate0.updateCounter(val);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long nextUpdateCounter() {
+ try {
+ CacheDataStore delegate0 = init0(false);
+
+ return delegate0 == null ? 0 : delegate0.nextUpdateCounter();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Long initialUpdateCounter() {
+ try {
+ CacheDataStore delegate0 = init0(true);
+
+ return delegate0 == null ? 0 : delegate0.initialUpdateCounter();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void updateInitialCounter(long cntr) {
+ try {
+ CacheDataStore delegate0 = init0(true);
+
+ if (delegate0 != null)
+ delegate0.updateInitialCounter(cntr);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void update(
+ GridCacheContext cctx,
+ KeyCacheObject key,
+ CacheObject val,
+ GridCacheVersion ver,
+ long expireTime,
+ @Nullable CacheDataRow oldRow
+ ) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(false);
+
+ delegate.update(cctx, key, val, ver, expireTime, oldRow);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void updateIndexes(GridCacheContext cctx, KeyCacheObject key) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(false);
+
+ delegate.updateIndexes(cctx, key);
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheDataRow createRow(
+ GridCacheContext cctx,
+ KeyCacheObject key,
+ CacheObject val,
+ GridCacheVersion ver,
+ long expireTime,
+ @Nullable CacheDataRow oldRow) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(false);
+
+ return delegate.createRow(cctx, key, val, ver, expireTime, oldRow);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void invoke(GridCacheContext cctx, KeyCacheObject key, OffheapInvokeClosure c)
+ throws IgniteCheckedException {
+ CacheDataStore delegate = init0(false);
+
+ delegate.invoke(cctx, key, c);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void remove(GridCacheContext cctx, KeyCacheObject key, int partId)
+ throws IgniteCheckedException {
+ CacheDataStore delegate = init0(false);
+
+ delegate.remove(cctx, key, partId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheDataRow find(GridCacheContext cctx, KeyCacheObject key) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(true);
+
+ if (delegate != null)
+ return delegate.find(cctx, key);
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCursor<? extends CacheDataRow> cursor() throws IgniteCheckedException {
+ CacheDataStore delegate = init0(true);
+
+ if (delegate != null)
+ return delegate.cursor();
+
+ return EMPTY_CURSOR;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCursor<? extends CacheDataRow> cursor(
+ int cacheId,
+ KeyCacheObject lower,
+ KeyCacheObject upper) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(true);
+
+ if (delegate != null)
+ return delegate.cursor(cacheId, lower, upper);
+
+ return EMPTY_CURSOR;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCursor<? extends CacheDataRow> cursor(int cacheId,
+ KeyCacheObject lower,
+ KeyCacheObject upper,
+ Object x)
+ throws IgniteCheckedException {
+ CacheDataStore delegate = init0(true);
+
+ if (delegate != null)
+ return delegate.cursor(cacheId, lower, upper, x);
+
+ return EMPTY_CURSOR;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void destroy() throws IgniteCheckedException {
+ // No need to destroy delegate.
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCursor<? extends CacheDataRow> cursor(int cacheId) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(true);
+
+ if (delegate != null)
+ return delegate.cursor(cacheId);
+
+ return EMPTY_CURSOR;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void clear(int cacheId) throws IgniteCheckedException {
+ CacheDataStore delegate = init0(true);
+
+ if (delegate != null)
+ delegate.clear(cacheId);
+ }
+ }
+
+ /**
+ *
+ */
+ private static final GridCursor<CacheDataRow> EMPTY_CURSOR = new GridCursor<CacheDataRow>() {
+ /** {@inheritDoc} */
+ @Override public boolean next() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheDataRow get() {
+ return null;
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
new file mode 100644
index 0000000..e45d379
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -0,0 +1,968 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import javax.management.JMException;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.MemoryMetrics;
+import org.apache.ignite.PersistenceMetrics;
+import org.apache.ignite.configuration.DataPageEvictionMode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.evict.FairFifoPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.evict.Random2LruPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.evict.RandomLruPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.mxbean.MemoryMetricsMXBean;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE;
+import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
+
+/**
+ *
+ */
+public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdapter
+ implements IgniteChangeGlobalStateSupport, CheckpointLockStateChecker {
+ /** MemoryPolicyConfiguration name reserved for internal caches. */
+ static final String SYSTEM_MEMORY_POLICY_NAME = "sysMemPlc";
+
+ /** Minimum size of memory chunk */
+ private static final long MIN_PAGE_MEMORY_SIZE = 10 * 1024 * 1024;
+
+ /** Maximum initial size on 32-bit JVM */
+ private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024;
+
+ /** */
+ protected Map<String, MemoryPolicy> memPlcMap;
+
+ /** */
+ protected Map<String, MemoryMetrics> memMetricsMap;
+
+ /** */
+ protected MemoryPolicy dfltMemPlc;
+
+ /** */
+ private Map<String, FreeListImpl> freeListMap;
+
+ /** */
+ private FreeListImpl dfltFreeList;
+
+ /** */
+ private int pageSize;
+
+ /** {@inheritDoc} */
+ @Override protected void start0() throws IgniteCheckedException {
+ if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getMemoryConfiguration() == null)
+ return;
+
+ init();
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void init() throws IgniteCheckedException {
+ MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration();
+
+ assert memCfg != null;
+
+ validateConfiguration(memCfg);
+
+ pageSize = memCfg.getPageSize();
+
+ initPageMemoryPolicies(memCfg);
+
+ //registerMetricsMBeans();
+
+ startMemoryPolicies();
+
+ initPageMemoryDataStructures(memCfg);
+ }
+
+ /**
+ * Registers MBeans for all MemoryMetrics configured in this instance.
+ */
+ private void registerMetricsMBeans() {
+ IgniteConfiguration cfg = cctx.gridConfig();
+
+ for (MemoryMetrics memMetrics : memMetricsMap.values()) {
+ MemoryPolicyConfiguration memPlcCfg = memPlcMap.get(memMetrics.getName()).config();
+
+ registerMetricsMBean((MemoryMetricsImpl)memMetrics, memPlcCfg, cfg);
+ }
+ }
+
+ /**
+ * @param memMetrics Memory metrics.
+ * @param memPlcCfg Memory policy configuration.
+ * @param cfg Ignite configuration.
+ */
+ private void registerMetricsMBean(
+ MemoryMetricsImpl memMetrics,
+ MemoryPolicyConfiguration memPlcCfg,
+ IgniteConfiguration cfg
+ ) {
+ try {
+ U.registerMBean(
+ cfg.getMBeanServer(),
+ cfg.getIgniteInstanceName(),
+ "MemoryMetrics",
+ memPlcCfg.getName(),
+ new MemoryMetricsMXBeanImpl(memMetrics, memPlcCfg),
+ MemoryMetricsMXBean.class);
+ }
+ catch (JMException e) {
+ U.error(log, "Failed to register MBean for MemoryMetrics with name: '" + memMetrics.getName() + "'", e);
+ }
+ }
+
+ /**
+ * @param dbCfg Database config.
+ */
+ protected void initPageMemoryDataStructures(MemoryConfiguration dbCfg) throws IgniteCheckedException {
+ freeListMap = U.newHashMap(memPlcMap.size());
+
+ String dfltMemPlcName = dbCfg.getDefaultMemoryPolicyName();
+
+ for (MemoryPolicy memPlc : memPlcMap.values()) {
+ MemoryPolicyConfiguration memPlcCfg = memPlc.config();
+
+ MemoryMetricsImpl memMetrics = (MemoryMetricsImpl) memMetricsMap.get(memPlcCfg.getName());
+
+ FreeListImpl freeList = new FreeListImpl(0,
+ cctx.igniteInstanceName(),
+ memMetrics,
+ memPlc,
+ null,
+ cctx.wal(),
+ 0L,
+ true);
+
+ memMetrics.freeList(freeList);
+
+ freeListMap.put(memPlcCfg.getName(), freeList);
+ }
+
+ dfltFreeList = freeListMap.get(dfltMemPlcName);
+ }
+
+ /**
+ * @return Size of page used for PageMemory regions.
+ */
+ public int pageSize() {
+ return pageSize;
+ }
+
+ /**
+ *
+ */
+ private void startMemoryPolicies() {
+ for (MemoryPolicy memPlc : memPlcMap.values()) {
+ memPlc.pageMemory().start();
+
+ memPlc.evictionTracker().start();
+ }
+ }
+
+ /**
+ * @param memCfg Database config.
+ */
+ protected void initPageMemoryPolicies(MemoryConfiguration memCfg) {
+ MemoryPolicyConfiguration[] memPlcsCfgs = memCfg.getMemoryPolicies();
+
+ if (memPlcsCfgs == null) {
+ //reserve place for default and system memory policies
+ memPlcMap = U.newHashMap(2);
+ memMetricsMap = U.newHashMap(2);
+
+ addMemoryPolicy(
+ memCfg,
+ memCfg.createDefaultPolicyConfig(),
+ DFLT_MEM_PLC_DEFAULT_NAME
+ );
+
+ U.warn(log, "No user-defined default MemoryPolicy found; system default of 1GB size will be used.");
+ }
+ else {
+ String dfltMemPlcName = memCfg.getDefaultMemoryPolicyName();
+
+ if (DFLT_MEM_PLC_DEFAULT_NAME.equals(dfltMemPlcName) && !hasCustomDefaultMemoryPolicy(memPlcsCfgs)) {
+ //reserve additional place for default and system memory policies
+ memPlcMap = U.newHashMap(memPlcsCfgs.length + 2);
+ memMetricsMap = U.newHashMap(memPlcsCfgs.length + 2);
+
+ addMemoryPolicy(
+ memCfg,
+ memCfg.createDefaultPolicyConfig(),
+ DFLT_MEM_PLC_DEFAULT_NAME
+ );
+
+ U.warn(log, "No user-defined default MemoryPolicy found; system default of 1GB size will be used.");
+ }
+ else {
+ //reserve additional space for system memory policy only
+ memPlcMap = U.newHashMap(memPlcsCfgs.length + 1);
+ memMetricsMap = U.newHashMap(memPlcsCfgs.length + 1);
+ }
+
+ for (MemoryPolicyConfiguration memPlcCfg : memPlcsCfgs)
+ addMemoryPolicy(memCfg, memPlcCfg, memPlcCfg.getName());
+ }
+
+ addMemoryPolicy(
+ memCfg,
+ createSystemMemoryPolicy(
+ memCfg.getSystemCacheInitialSize(),
+ memCfg.getSystemCacheMaxSize()
+ ),
+ SYSTEM_MEMORY_POLICY_NAME
+ );
+ }
+
+ /**
+ * @param memCfg Database config.
+ * @param memPlcCfg Memory policy config.
+ * @param memPlcName Memory policy name.
+ */
+ private void addMemoryPolicy(
+ MemoryConfiguration memCfg,
+ MemoryPolicyConfiguration memPlcCfg,
+ String memPlcName
+ ) {
+ String dfltMemPlcName = memCfg.getDefaultMemoryPolicyName();
+
+ if (dfltMemPlcName == null)
+ dfltMemPlcName = DFLT_MEM_PLC_DEFAULT_NAME;
+
+ MemoryMetricsImpl memMetrics = new MemoryMetricsImpl(memPlcCfg);
+
+ MemoryPolicy memPlc = initMemory(memCfg, memPlcCfg, memMetrics);
+
+ memPlcMap.put(memPlcName, memPlc);
+
+ memMetricsMap.put(memPlcName, memMetrics);
+
+ if (memPlcName.equals(dfltMemPlcName))
+ dfltMemPlc = memPlc;
+ else if (memPlcName.equals(DFLT_MEM_PLC_DEFAULT_NAME))
+ U.warn(log, "Memory Policy with name 'default' isn't used as a default. " +
+ "Please check Memory Policies configuration.");
+ }
+
+ /**
+ * @param memPlcsCfgs User-defined memory policy configurations.
+ */
+ private boolean hasCustomDefaultMemoryPolicy(MemoryPolicyConfiguration[] memPlcsCfgs) {
+ for (MemoryPolicyConfiguration memPlcsCfg : memPlcsCfgs) {
+ if (DFLT_MEM_PLC_DEFAULT_NAME.equals(memPlcsCfg.getName()))
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param dbCfg Database configuration.
+ * @param memPlcCfg MemoryPolicy configuration.
+ * @param memMetrics MemoryMetrics instance.
+ */
+ private MemoryPolicy createDefaultMemoryPolicy(MemoryConfiguration dbCfg, MemoryPolicyConfiguration memPlcCfg, MemoryMetricsImpl memMetrics) {
+ return initMemory(dbCfg, memPlcCfg, memMetrics);
+ }
+
+ /**
+ * @param sysCacheInitSize Initial size of PageMemory to be created for system cache.
+ * @param sysCacheMaxSize Maximum size of PageMemory to be created for system cache.
+ *
+ * @return {@link MemoryPolicyConfiguration configuration} of MemoryPolicy for system cache.
+ */
+ private MemoryPolicyConfiguration createSystemMemoryPolicy(long sysCacheInitSize, long sysCacheMaxSize) {
+ MemoryPolicyConfiguration res = new MemoryPolicyConfiguration();
+
+ res.setName(SYSTEM_MEMORY_POLICY_NAME);
+ res.setInitialSize(sysCacheInitSize);
+ res.setMaxSize(sysCacheMaxSize);
+
+ return res;
+ }
+
+ /**
+ * @param memCfg configuration to validate.
+ */
+ private void validateConfiguration(MemoryConfiguration memCfg) throws IgniteCheckedException {
+ MemoryPolicyConfiguration[] plcCfgs = memCfg.getMemoryPolicies();
+
+ Set<String> plcNames = (plcCfgs != null) ? U.<String>newHashSet(plcCfgs.length) : new HashSet<String>(0);
+
+ checkSystemMemoryPolicySizeConfiguration(
+ memCfg.getSystemCacheInitialSize(),
+ memCfg.getSystemCacheMaxSize()
+ );
+
+ if (plcCfgs != null) {
+ for (MemoryPolicyConfiguration plcCfg : plcCfgs) {
+ assert plcCfg != null;
+
+ checkPolicyName(plcCfg.getName(), plcNames);
+
+ checkPolicySize(plcCfg);
+
+ checkMetricsProperties(plcCfg);
+
+ checkPolicyEvictionProperties(plcCfg, memCfg);
+ }
+ }
+
+ checkDefaultPolicyConfiguration(
+ memCfg.getDefaultMemoryPolicyName(),
+ memCfg.getDefaultMemoryPolicySize(),
+ plcNames
+ );
+ }
+
+ /**
+ * @param plcCfg Memory policy config.
+ *
+ * @throws IgniteCheckedException if validation of memory metrics properties fails.
+ */
+ private static void checkMetricsProperties(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException {
+ if (plcCfg.getRateTimeInterval() <= 0)
+ throw new IgniteCheckedException("Rate time interval must be greater than zero " +
+ "(use MemoryPolicyConfiguration.rateTimeInterval property to adjust the interval) " +
+ "[name=" + plcCfg.getName() +
+ ", rateTimeInterval=" + plcCfg.getRateTimeInterval() + "]"
+ );
+ if (plcCfg.getSubIntervals() <= 0)
+ throw new IgniteCheckedException("Sub intervals must be greater than zero " +
+ "(use MemoryPolicyConfiguration.subIntervals property to adjust the sub intervals) " +
+ "[name=" + plcCfg.getName() +
+ ", subIntervals=" + plcCfg.getSubIntervals() + "]"
+ );
+
+ if (plcCfg.getRateTimeInterval() < 1_000)
+ throw new IgniteCheckedException("Rate time interval must be longer that 1 second (1_000 milliseconds) " +
+ "(use MemoryPolicyConfiguration.rateTimeInterval property to adjust the interval) " +
+ "[name=" + plcCfg.getName() +
+ ", rateTimeInterval=" + plcCfg.getRateTimeInterval() + "]");
+ }
+
+ /**
+ * @param sysCacheInitSize System cache initial size.
+ * @param sysCacheMaxSize System cache max size.
+ *
+ * @throws IgniteCheckedException In case of validation violation.
+ */
+ private static void checkSystemMemoryPolicySizeConfiguration(
+ long sysCacheInitSize,
+ long sysCacheMaxSize
+ ) throws IgniteCheckedException {
+ if (sysCacheInitSize < MIN_PAGE_MEMORY_SIZE)
+ throw new IgniteCheckedException("Initial size for system cache must have size more than 10MB (use " +
+ "MemoryConfiguration.systemCacheInitialSize property to set correct size in bytes) " +
+ "[size=" + U.readableSize(sysCacheInitSize, true) + ']'
+ );
+
+ if (U.jvm32Bit() && sysCacheInitSize > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT)
+ throw new IgniteCheckedException("Initial size for system cache exceeds 2GB on 32-bit JVM (use " +
+ "MemoryPolicyConfiguration.systemCacheInitialSize property to set correct size in bytes " +
+ "or use 64-bit JVM) [size=" + U.readableSize(sysCacheInitSize, true) + ']'
+ );
+
+ if (sysCacheMaxSize < sysCacheInitSize)
+ throw new IgniteCheckedException("MaxSize of system cache must not be smaller than " +
+ "initialSize [initSize=" + U.readableSize(sysCacheInitSize, true) +
+ ", maxSize=" + U.readableSize(sysCacheMaxSize, true) + "]. " +
+ "Use MemoryConfiguration.systemCacheInitialSize/MemoryConfiguration.systemCacheMaxSize " +
+ "properties to set correct sizes in bytes."
+ );
+ }
+
+ /**
+ * @param dfltPlcName Default MemoryPolicy name.
+ * @param dfltPlcSize Default size of MemoryPolicy overridden by user (equals to -1 if wasn't specified by user).
+ * @param plcNames All MemoryPolicy names.
+ * @throws IgniteCheckedException In case of validation violation.
+ */
+ private static void checkDefaultPolicyConfiguration(
+ String dfltPlcName,
+ long dfltPlcSize,
+ Collection<String> plcNames
+ ) throws IgniteCheckedException {
+ if (dfltPlcSize != MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE) {
+ if (!F.eq(dfltPlcName, MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME))
+ throw new IgniteCheckedException("User-defined MemoryPolicy configuration " +
+ "and defaultMemoryPolicySize properties are set at the same time. " +
+ "Delete either MemoryConfiguration.defaultMemoryPolicySize property " +
+ "or user-defined default MemoryPolicy configuration");
+
+ if (dfltPlcSize < MIN_PAGE_MEMORY_SIZE)
+ throw new IgniteCheckedException("User-defined default MemoryPolicy size is less than 1MB. " +
+ "Use MemoryConfiguration.defaultMemoryPolicySize property to set correct size.");
+
+ if (U.jvm32Bit() && dfltPlcSize > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT)
+ throw new IgniteCheckedException("User-defined default MemoryPolicy size exceeds 2GB on 32-bit JVM " +
+ "(use MemoryConfiguration.defaultMemoryPolicySize property to set correct size in bytes " +
+ "or use 64-bit JVM) [size=" + U.readableSize(dfltPlcSize, true) + ']'
+ );
+ }
+
+ if (!DFLT_MEM_PLC_DEFAULT_NAME.equals(dfltPlcName)) {
+ if (dfltPlcName.isEmpty())
+ throw new IgniteCheckedException("User-defined default MemoryPolicy name must be non-empty");
+
+ if (!plcNames.contains(dfltPlcName))
+ throw new IgniteCheckedException("User-defined default MemoryPolicy name " +
+ "must be presented among configured MemoryPolices: " + dfltPlcName);
+ }
+ }
+
+ /**
+ * @param plcCfg MemoryPolicyConfiguration to validate.
+ * @throws IgniteCheckedException If config is invalid.
+ */
+ private void checkPolicySize(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException {
+ if (plcCfg.getInitialSize() < MIN_PAGE_MEMORY_SIZE)
+ throw new IgniteCheckedException("MemoryPolicy must have size more than 10MB (use " +
+ "MemoryPolicyConfiguration.initialSize property to set correct size in bytes) " +
+ "[name=" + plcCfg.getName() + ", size=" + U.readableSize(plcCfg.getInitialSize(), true) + "]"
+ );
+
+ if (plcCfg.getMaxSize() < plcCfg.getInitialSize()) {
+ // We will know for sure if initialSize has been changed if we compare Longs by "==".
+ if (plcCfg.getInitialSize() == DFLT_MEMORY_POLICY_INITIAL_SIZE) {
+ plcCfg.setInitialSize(plcCfg.getMaxSize());
+
+ LT.warn(log, "MemoryPolicy maxSize=" + U.readableSize(plcCfg.getMaxSize(), true) +
+ " is smaller than defaultInitialSize=" +
+ U.readableSize(MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE, true) +
+ ", setting initialSize to " + U.readableSize(plcCfg.getMaxSize(), true));
+ }
+ else {
+ throw new IgniteCheckedException("MemoryPolicy maxSize must not be smaller than " +
+ "initialSize [name=" + plcCfg.getName() +
+ ", initSize=" + U.readableSize(plcCfg.getInitialSize(), true) +
+ ", maxSize=" + U.readableSize(plcCfg.getMaxSize(), true) + ']');
+ }
+ }
+
+ if (U.jvm32Bit() && plcCfg.getInitialSize() > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT)
+ throw new IgniteCheckedException("MemoryPolicy initialSize exceeds 2GB on 32-bit JVM (use " +
+ "MemoryPolicyConfiguration.initialSize property to set correct size in bytes or use 64-bit JVM) " +
+ "[name=" + plcCfg.getName() +
+ ", size=" + U.readableSize(plcCfg.getInitialSize(), true) + "]");
+ }
+
+ /**
+ * @param plcCfg MemoryPolicyConfiguration to validate.
+ * @param dbCfg Memory configuration.
+ * @throws IgniteCheckedException If config is invalid.
+ */
+ protected void checkPolicyEvictionProperties(MemoryPolicyConfiguration plcCfg, MemoryConfiguration dbCfg)
+ throws IgniteCheckedException {
+ if (plcCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED)
+ return;
+
+ if (plcCfg.getEvictionThreshold() < 0.5 || plcCfg.getEvictionThreshold() > 0.999) {
+ throw new IgniteCheckedException("Page eviction threshold must be between 0.5 and 0.999: " +
+ plcCfg.getName());
+ }
+
+ if (plcCfg.getEmptyPagesPoolSize() <= 10)
+ throw new IgniteCheckedException("Evicted pages pool size should be greater than 10: " + plcCfg.getName());
+
+ long maxPoolSize = plcCfg.getMaxSize() / dbCfg.getPageSize() / 10;
+
+ if (plcCfg.getEmptyPagesPoolSize() >= maxPoolSize) {
+ throw new IgniteCheckedException("Evicted pages pool size should be lesser than " + maxPoolSize +
+ ": " + plcCfg.getName());
+ }
+ }
+
+ /**
+ * @param plcName MemoryPolicy name to validate.
+ * @param observedNames Names of MemoryPolicies observed before.
+ * @throws IgniteCheckedException If config is invalid.
+ */
+ private static void checkPolicyName(String plcName, Collection<String> observedNames)
+ throws IgniteCheckedException {
+ if (plcName == null || plcName.isEmpty())
+ throw new IgniteCheckedException("User-defined MemoryPolicyConfiguration must have non-null and " +
+ "non-empty name.");
+
+ if (observedNames.contains(plcName))
+ throw new IgniteCheckedException("Two MemoryPolicies have the same name: " + plcName);
+
+ if (SYSTEM_MEMORY_POLICY_NAME.equals(plcName))
+ throw new IgniteCheckedException("'sysMemPlc' policy name is reserved for internal use.");
+
+ observedNames.add(plcName);
+ }
+
+ /**
+ * @param log Logger.
+ */
+ public void dumpStatistics(IgniteLogger log) {
+ if (freeListMap != null) {
+ for (FreeListImpl freeList : freeListMap.values())
+ freeList.dumpStatistics(log);
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void initDataBase() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /**
+ * @return collection of all configured {@link MemoryPolicy policies}.
+ */
+ public Collection<MemoryPolicy> memoryPolicies() {
+ return memPlcMap != null ? memPlcMap.values() : null;
+ }
+
+ /**
+ * @return MemoryMetrics for all MemoryPolicies configured in Ignite instance.
+ */
+ public Collection<MemoryMetrics> memoryMetrics() {
+ if (!F.isEmpty(memMetricsMap)) {
+ // Intentionally return a collection copy to make it explicitly serializable.
+ Collection<MemoryMetrics> res = new ArrayList<>(memMetricsMap.size());
+
+ for (MemoryMetrics metrics : memMetricsMap.values())
+ res.add(new MemoryMetricsSnapshot(metrics));
+
+ return res;
+ }
+ else
+ return Collections.emptyList();
+ }
+
+ /**
+ * @return PersistenceMetrics if persistence is enabled or {@code null} otherwise.
+ */
+ public PersistenceMetrics persistentStoreMetrics() {
+ return null;
+ }
+
+ /**
+ * @param memPlcName Name of {@link MemoryPolicy} to obtain {@link MemoryMetrics} for.
+ * @return {@link MemoryMetrics} snapshot for specified {@link MemoryPolicy} or {@code null} if
+ * no {@link MemoryPolicy} is configured for specified name.
+ */
+ @Nullable public MemoryMetrics memoryMetrics(String memPlcName) {
+ if (!F.isEmpty(memMetricsMap)) {
+ MemoryMetrics memMetrics = memMetricsMap.get(memPlcName);
+
+ if (memMetrics == null)
+ return null;
+ else
+ return new MemoryMetricsSnapshot(memMetrics);
+ }
+ else
+ return null;
+ }
+
+ /**
+ * @param memPlcName Memory policy name.
+ * @return {@link MemoryPolicy} instance associated with a given {@link MemoryPolicyConfiguration}.
+ * @throws IgniteCheckedException in case of request for unknown MemoryPolicy.
+ */
+ public MemoryPolicy memoryPolicy(String memPlcName) throws IgniteCheckedException {
+ if (memPlcName == null)
+ return dfltMemPlc;
+
+ if (memPlcMap == null)
+ return null;
+
+ MemoryPolicy plc;
+
+ if ((plc = memPlcMap.get(memPlcName)) == null)
+ throw new IgniteCheckedException("Requested MemoryPolicy is not configured: " + memPlcName);
+
+ return plc;
+ }
+
+ /**
+ * @param memPlcName MemoryPolicyConfiguration name.
+ * @return {@link FreeList} instance associated with a given {@link MemoryPolicyConfiguration}.
+ */
+ public FreeList freeList(String memPlcName) {
+ if (memPlcName == null)
+ return dfltFreeList;
+
+ return freeListMap != null ? freeListMap.get(memPlcName) : null;
+ }
+
+ /**
+ * @param memPlcName MemoryPolicyConfiguration name.
+ * @return {@link ReuseList} instance associated with a given {@link MemoryPolicyConfiguration}.
+ */
+ public ReuseList reuseList(String memPlcName) {
+ if (memPlcName == null)
+ return dfltFreeList;
+
+ return freeListMap != null ? freeListMap.get(memPlcName) : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void stop0(boolean cancel) {
+ if (memPlcMap != null) {
+ for (MemoryPolicy memPlc : memPlcMap.values()) {
+ memPlc.pageMemory().stop();
+
+ memPlc.evictionTracker().stop();
+
+ IgniteConfiguration cfg = cctx.gridConfig();
+
+ /* try {
+ cfg.getMBeanServer().unregisterMBean(
+ U.makeMBeanName(
+ cfg.getIgniteInstanceName(),
+ "MemoryMetrics",
+ memPlc.memoryMetrics().getName()));
+ }
+ catch (JMException e) {
+ U.error(log, "Failed to unregister MBean for memory metrics: " +
+ memPlc.memoryMetrics().getName(), e);
+ }*/
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ public boolean persistenceEnabled() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return false;
+ }
+
+ /**
+ *
+ */
+ public void lock() throws IgniteCheckedException {
+
+ }
+
+ /**
+ *
+ */
+ public void unLock() {
+
+ }
+
+ /**
+ * No-op for non-persistent storage.
+ */
+ public void checkpointReadLock() {
+ // No-op.
+ }
+
+ /**
+ * No-op for non-persistent storage.
+ */
+ public void checkpointReadUnlock() {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ @Nullable public IgniteInternalFuture wakeupForCheckpoint(String reason) {
+ return null;
+ }
+
+ /**
+ * Waits until current state is checkpointed.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void waitForCheckpoint(String reason) throws IgniteCheckedException {
+ // No-op
+ }
+
+ /**
+ * @param discoEvt Before exchange for the given discovery event.
+ */
+ public void beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /**
+ * Needed action before any cache will stop
+ */
+ public void prepareCachesStop() {
+ // No-op.
+ }
+
+ /**
+ * @param stoppedGrps A collection of tuples (cache group, destroy flag).
+ */
+ public void onCacheGroupsStopped(Collection<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGrps) {
+ // No-op.
+ }
+
+ /**
+ * @param cctx Stopped cache context.
+ */
+ public void onCacheStop(GridCacheContext cctx) {
+ // No-op
+ }
+
+ /**
+ * @return Future that will be completed when indexes for given cache are restored.
+ */
+ @Nullable public IgniteInternalFuture indexRebuildFuture(int cacheId) {
+ return null;
+ }
+
+ /**
+ * Reserve update history for exchange.
+ *
+ * @return Reserved update counters per cache and partition.
+ */
+ public Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
+ return Collections.emptyMap();
+ }
+
+ /**
+ * Release reserved update history.
+ */
+ public void releaseHistoryForExchange() {
+ // No-op
+ }
+
+ /**
+ * Reserve update history for preloading.
+ * @param grpId Cache group ID.
+ * @param partId Partition Id.
+ * @param cntr Update counter.
+ * @return True if successfully reserved.
+ */
+ public boolean reserveHistoryForPreloading(int grpId, int partId, long cntr) {
+ return false;
+ }
+
+ /**
+ * Release reserved update history.
+ */
+ public void releaseHistoryForPreloading() {
+ // No-op
+ }
+
+ /**
+ * See {@link GridCacheMapEntry#ensureFreeSpace()}
+ *
+ * @param memPlc Memory policy.
+ */
+ public void ensureFreeSpace(MemoryPolicy memPlc) throws IgniteCheckedException {
+ if (memPlc == null)
+ return;
+
+ MemoryPolicyConfiguration plcCfg = memPlc.config();
+
+ if (plcCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED)
+ return;
+
+ long memorySize = plcCfg.getMaxSize();
+
+ PageMemory pageMem = memPlc.pageMemory();
+
+ int sysPageSize = pageMem.systemPageSize();
+
+ FreeListImpl freeListImpl = freeListMap.get(plcCfg.getName());
+
+ for (;;) {
+ long allocatedPagesCnt = pageMem.loadedPages();
+
+ int emptyDataPagesCnt = freeListImpl.emptyDataPages();
+
+ boolean shouldEvict = allocatedPagesCnt > (memorySize / sysPageSize * plcCfg.getEvictionThreshold()) &&
+ emptyDataPagesCnt < plcCfg.getEmptyPagesPoolSize();
+
+ if (shouldEvict)
+ memPlc.evictionTracker().evictDataPage();
+ else
+ break;
+ }
+ }
+
+ /**
+ * @param memCfg memory configuration with common parameters.
+ * @param plcCfg memory policy with PageMemory specific parameters.
+ * @param memMetrics {@link MemoryMetrics} object to collect memory usage metrics.
+ * @return Memory policy instance.
+ */
+ private MemoryPolicy initMemory(
+ MemoryConfiguration memCfg,
+ MemoryPolicyConfiguration plcCfg,
+ MemoryMetricsImpl memMetrics
+ ) {
+ File allocPath = buildAllocPath(plcCfg);
+
+ DirectMemoryProvider memProvider = allocPath == null ?
+ new UnsafeMemoryProvider(log) :
+ new MappedFileMemoryProvider(
+ log,
+ allocPath);
+
+ PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
+
+ return new MemoryPolicy(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
+ }
+
+ /**
+ * @param plc Memory Policy Configuration.
+ * @param pageMem Page memory.
+ */
+ private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration plc, PageMemory pageMem) {
+ if (plc.getPageEvictionMode() == DataPageEvictionMode.DISABLED)
+ return new NoOpPageEvictionTracker();
+
+ assert pageMem instanceof PageMemoryNoStoreImpl : pageMem.getClass();
+
+ PageMemoryNoStoreImpl pageMem0 = (PageMemoryNoStoreImpl)pageMem;
+
+ if (Boolean.getBoolean("override.fair.fifo.page.eviction.tracker"))
+ return new FairFifoPageEvictionTracker(pageMem0, plc, cctx);
+
+ switch (plc.getPageEvictionMode()) {
+ case RANDOM_LRU:
+ return new RandomLruPageEvictionTracker(pageMem0, plc, cctx);
+ case RANDOM_2_LRU:
+ return new Random2LruPageEvictionTracker(pageMem0, plc, cctx);
+ default:
+ return new NoOpPageEvictionTracker();
+ }
+ }
+
+ /**
+ * Builds allocation path for memory mapped file to be used with PageMemory.
+ *
+ * @param plc MemoryPolicyConfiguration.
+ */
+ @Nullable protected File buildAllocPath(MemoryPolicyConfiguration plc) {
+ String path = plc.getSwapFilePath();
+
+ if (path == null)
+ return null;
+
+ String consId = String.valueOf(cctx.discovery().consistentId());
+
+ consId = consId.replaceAll("[:,\\.]", "_");
+
+ return buildPath(path, consId);
+ }
+
+ /**
+ * Creates PageMemory with given size and memory provider.
+ *
+ * @param memProvider Memory provider.
+ * @param memCfg Memory configuartion.
+ * @param memPlcCfg Memory policy configuration.
+ * @param memMetrics MemoryMetrics to collect memory usage metrics.
+ * @return PageMemory instance.
+ */
+ protected PageMemory createPageMemory(
+ DirectMemoryProvider memProvider,
+ MemoryConfiguration memCfg,
+ MemoryPolicyConfiguration memPlcCfg,
+ MemoryMetricsImpl memMetrics
+ ) {
+ memMetrics.persistenceEnabled(false);
+
+ return new PageMemoryNoStoreImpl(
+ log,
+ memProvider,
+ cctx,
+ memCfg.getPageSize(),
+ memPlcCfg,
+ memMetrics,
+ false
+ );
+ }
+
+ /**
+ * @param path Path to the working directory.
+ * @param consId Consistent ID of the local node.
+ * @return DB storage path.
+ */
+ protected File buildPath(String path, String consId) {
+ String igniteHomeStr = U.getIgniteHome();
+
+ File igniteHome = igniteHomeStr != null ? new File(igniteHomeStr) : null;
+
+ File workDir = igniteHome == null ? new File(path) : new File(igniteHome, path);
+
+ return new File(workDir, consId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ start0();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ stop0(true);
+ }
+
+ /**
+ * @return Name of MemoryPolicyConfiguration for internal caches.
+ */
+ public String systemMemoryPolicyName() {
+ return SYSTEM_MEMORY_POLICY_NAME;
+ }
+}
[28/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
new file mode 100644
index 0000000..4ec1efd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -0,0 +1,2533 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.LockSupport;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.regex.Pattern;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.PersistenceMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
+import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.RecordV1Serializer;
+import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * File WAL manager.
+ */
+public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter implements IgniteWriteAheadLogManager {
+ /** */
+ public static final FileDescriptor[] EMPTY_DESCRIPTORS = new FileDescriptor[0];
+
+ /** */
+ public static final String WAL_SEGMENT_FILE_EXT = ".wal";
+
+ /** */
+ private static final byte[] FILL_BUF = new byte[1024 * 1024];
+
+ /** */
+ private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
+
+ /** */
+ private static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp");
+
+ /** */
+ private static final FileFilter WAL_SEGMENT_FILE_FILTER = new FileFilter() {
+ @Override public boolean accept(File file) {
+ return !file.isDirectory() && WAL_NAME_PATTERN.matcher(file.getName()).matches();
+ }
+ };
+
+ /** */
+ private static final FileFilter WAL_SEGMENT_TEMP_FILE_FILTER = new FileFilter() {
+ @Override public boolean accept(File file) {
+ return !file.isDirectory() && WAL_TEMP_NAME_PATTERN.matcher(file.getName()).matches();
+ }
+ };
+
+ /** */
+ private final boolean alwaysWriteFullPages;
+
+ /** WAL segment size in bytes */
+ private final long maxWalSegmentSize;
+
+ /** */
+ private final WALMode mode;
+
+ /** Thread local byte buffer size, see {@link #tlb} */
+ private final int tlbSize;
+
+ /** WAL flush frequency. Makes sense only for {@link WALMode#BACKGROUND} log WALMode. */
+ public final int flushFreq;
+
+ /** Fsync delay. */
+ private final long fsyncDelay;
+
+ /** */
+ private final PersistentStoreConfiguration psCfg;
+
+ /** */
+ private IgniteConfiguration igCfg;
+
+ /** Persistence metrics tracker. */
+ private PersistenceMetricsImpl metrics;
+
+ /** */
+ private File walWorkDir;
+
+ /** */
+ private File walArchiveDir;
+
+ /** */
+ private RecordSerializer serializer;
+
+ /** */
+ private volatile long oldestArchiveSegmentIdx;
+
+ /** Updater for {@link #currentHnd}, used for verify there are no concurrent update for current log segment handle */
+ private static final AtomicReferenceFieldUpdater<FileWriteAheadLogManager, FileWriteHandle> currentHndUpd =
+ AtomicReferenceFieldUpdater.newUpdater(FileWriteAheadLogManager.class, FileWriteHandle.class, "currentHnd");
+
+ /**
+ * Thread local byte buffer for saving serialized WAL records chain, see {@link FileWriteHandle#head}.
+ * Introduced to decrease number of buffers allocation.
+ * Used only for record itself is shorter than {@link #tlbSize}.
+ */
+ private final ThreadLocal<ByteBuffer> tlb = new ThreadLocal<ByteBuffer>() {
+ @Override protected ByteBuffer initialValue() {
+ ByteBuffer buf = ByteBuffer.allocateDirect(tlbSize);
+
+ buf.order(GridUnsafe.NATIVE_BYTE_ORDER);
+
+ return buf;
+ }
+ };
+
+ /** */
+ private volatile FileArchiver archiver;
+
+ /** */
+ private QueueFlusher flusher;
+
+ /** */
+ private final ThreadLocal<WALPointer> lastWALPtr = new ThreadLocal<>();
+
+ /** Current log segment handle */
+ private volatile FileWriteHandle currentHnd;
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public FileWriteAheadLogManager(GridKernalContext ctx) {
+ igCfg = ctx.config();
+
+ PersistentStoreConfiguration psCfg = igCfg.getPersistentStoreConfiguration();
+
+ assert psCfg != null : "WAL should not be created if persistence is disabled.";
+
+ this.psCfg = psCfg;
+
+ maxWalSegmentSize = psCfg.getWalSegmentSize();
+ mode = psCfg.getWalMode();
+ tlbSize = psCfg.getTlbSize();
+ flushFreq = psCfg.getWalFlushFrequency();
+ fsyncDelay = psCfg.getWalFsyncDelay();
+ alwaysWriteFullPages = psCfg.isAlwaysWriteFullPages();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start0() throws IgniteCheckedException {
+ if (!cctx.kernalContext().clientNode()) {
+ String consId = consistentId();
+
+ A.notNullOrEmpty(consId, "consistentId");
+
+ consId = U.maskForFileName(consId);
+
+ checkWalConfiguration();
+
+ walWorkDir = initDirectory(
+ psCfg.getWalStorePath(),
+ PersistentStoreConfiguration.DFLT_WAL_STORE_PATH,
+ consId,
+ "write ahead log work directory"
+ );
+
+ walArchiveDir = initDirectory(
+ psCfg.getWalArchivePath(),
+ PersistentStoreConfiguration.DFLT_WAL_ARCHIVE_PATH,
+ consId,
+ "write ahead log archive directory"
+ );
+
+ serializer = new RecordV1Serializer(cctx);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.database();
+
+ metrics = dbMgr.persistentStoreMetricsImpl();
+
+ checkOrPrepareFiles();
+
+ IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
+
+ oldestArchiveSegmentIdx = tup == null ? 0 : tup.get1();
+
+ archiver = new FileArchiver(tup == null ? -1 : tup.get2());
+
+ if (mode != WALMode.DEFAULT) {
+ if (log.isInfoEnabled())
+ log.info("Started write-ahead log manager [mode=" + mode + ']');
+ }
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException if WAL store path is configured and archive path isn't (or vice versa)
+ */
+ private void checkWalConfiguration() throws IgniteCheckedException {
+ if (psCfg.getWalStorePath() == null ^ psCfg.getWalArchivePath() == null) {
+ throw new IgniteCheckedException(
+ "Properties should be either both specified or both null " +
+ "[walStorePath = " + psCfg.getWalStorePath() +
+ ", walArchivePath = " + psCfg.getWalArchivePath() + "]"
+ );
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException {
+ super.onKernalStart0(reconnect);
+
+ if (!cctx.kernalContext().clientNode() && cctx.kernalContext().state().active())
+ archiver.start();
+ }
+
+ /**
+ * @return Consistent ID.
+ */
+ protected String consistentId() {
+ return cctx.discovery().consistentId().toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void stop0(boolean cancel) {
+ FileWriteHandle currentHnd = currentHandle();
+
+ try {
+ QueueFlusher flusher0 = flusher;
+
+ if (flusher0 != null) {
+ flusher0.shutdown();
+
+ if (currentHnd != null)
+ currentHnd.flush((FileWALPointer)null);
+ }
+
+ if (currentHnd != null)
+ currentHnd.close(false);
+
+ if (archiver != null)
+ archiver.shutdown();
+ }
+ catch (Exception e) {
+ U.error(log, "Failed to gracefully close WAL segment: " + currentHnd.file, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Activated file write ahead log manager [nodeId=" + cctx.localNodeId() +
+ " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
+
+ start0();
+
+ if (!cctx.kernalContext().clientNode()) {
+ assert archiver != null;
+
+ archiver.start();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDeActivate(GridKernalContext kctx) {
+ if (log.isDebugEnabled())
+ log.debug("DeActivate file write ahead log [nodeId=" + cctx.localNodeId() +
+ " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
+
+ stop0(true);
+
+ currentHnd = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isAlwaysWriteFullPages() {
+ return alwaysWriteFullPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isFullSync() {
+ return mode == WALMode.DEFAULT;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
+ try {
+ assert currentHnd == null;
+ assert lastPtr == null || lastPtr instanceof FileWALPointer;
+
+ FileWALPointer filePtr = (FileWALPointer)lastPtr;
+
+ currentHnd = restoreWriteHandle(filePtr);
+
+ if (currentHnd.serializer.version() != serializer.version()) {
+ if (log.isInfoEnabled())
+ log.info("Record serializer version change detected, will start logging with a new WAL record " +
+ "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
+ ", oldVer=" + currentHnd.serializer.version() + ']');
+
+ rollOver(currentHnd);
+ }
+
+ if (mode == WALMode.BACKGROUND) {
+ flusher = new QueueFlusher(cctx.igniteInstanceName());
+
+ flusher.start();
+ }
+ }
+ catch (StorageException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("TooBroadScope")
+ @Override public WALPointer log(WALRecord record) throws IgniteCheckedException, StorageException {
+ if (serializer == null || mode == WALMode.NONE)
+ return null;
+
+ FileWriteHandle current = currentHandle();
+
+ // Logging was not resumed yet.
+ if (current == null)
+ return null;
+
+ // Need to calculate record size first.
+ record.size(serializer.size(record));
+
+ for (; ; current = rollOver(current)) {
+ WALPointer ptr = current.addRecord(record);
+
+ if (ptr != null) {
+ metrics.onWalRecordLogged();
+
+ lastWALPtr.set(ptr);
+
+ return ptr;
+ }
+
+ if (isStopping())
+ throw new IgniteCheckedException("Stopping.");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void fsync(WALPointer ptr) throws IgniteCheckedException, StorageException {
+ if (serializer == null || mode == WALMode.NONE)
+ return;
+
+ FileWriteHandle cur = currentHandle();
+
+ // WAL manager was not started (client node).
+ if (cur == null)
+ return;
+
+ FileWALPointer filePtr = (FileWALPointer)(ptr == null ? lastWALPtr.get() : ptr);
+
+ boolean forceFlush = filePtr != null && filePtr.forceFlush();
+
+ if (mode == WALMode.BACKGROUND && !forceFlush)
+ return;
+
+ if (mode == WALMode.LOG_ONLY || forceFlush) {
+ cur.flushOrWait(filePtr);
+
+ return;
+ }
+
+ // No need to sync if was rolled over.
+ if (filePtr != null && !cur.needFsync(filePtr))
+ return;
+
+ cur.fsync(filePtr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALIterator replay(WALPointer start)
+ throws IgniteCheckedException, StorageException {
+ assert start == null || start instanceof FileWALPointer : "Invalid start pointer: " + start;
+
+ FileWriteHandle hnd = currentHandle();
+
+ FileWALPointer end = null;
+
+ if (hnd != null)
+ end = hnd.position();
+
+ return new RecordsIterator(
+ cctx,
+ walWorkDir,
+ walArchiveDir,
+ (FileWALPointer)start,
+ end,
+ psCfg,
+ serializer,
+ archiver,
+ log,
+ tlbSize
+ );
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+ assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
+
+ if (mode == WALMode.NONE)
+ return false;
+
+ FileArchiver archiver0 = archiver;
+
+ if (archiver0 == null)
+ throw new IgniteCheckedException("Could not reserve WAL segment: archiver == null");
+
+ archiver0.reserve(((FileWALPointer)start).index());
+
+ if (!hasIndex(((FileWALPointer)start).index())) {
+ archiver0.release(((FileWALPointer)start).index());
+
+ return false;
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void release(WALPointer start) throws IgniteCheckedException {
+ assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
+
+ if (mode == WALMode.NONE)
+ return;
+
+ FileArchiver archiver0 = archiver;
+
+ if (archiver0 == null)
+ throw new IgniteCheckedException("Could not release WAL segment: archiver == null");
+
+ archiver0.release(((FileWALPointer)start).index());
+ }
+
+ /**
+ * @param absIdx Absolulte index to check.
+ * @return {@code true} if has this index.
+ */
+ private boolean hasIndex(long absIdx) {
+ String name = FileDescriptor.fileName(absIdx);
+
+ boolean inArchive = new File(walArchiveDir, name).exists();
+
+ if (inArchive)
+ return true;
+
+ if (absIdx <= lastArchivedIndex())
+ return false;
+
+ FileWriteHandle cur = currentHnd;
+
+ return cur != null && cur.idx >= absIdx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int truncate(WALPointer ptr) {
+ if (ptr == null)
+ return 0;
+
+ assert ptr instanceof FileWALPointer : ptr;
+
+ FileWALPointer fPtr = (FileWALPointer)ptr;
+
+ FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER));
+
+ int deleted = 0;
+
+ FileArchiver archiver0 = archiver;
+
+ for (FileDescriptor desc : descs) {
+ // Do not delete reserved segment and any segment after it.
+ if (archiver0 != null && archiver0.reserved(desc.idx))
+ return deleted;
+
+ // We need to leave at least one archived segment to correctly determine the archive index.
+ if (desc.idx + 1 < fPtr.index()) {
+ if (!desc.file.delete())
+ U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
+ desc.file.getAbsolutePath());
+ else
+ deleted++;
+
+ // Bump up the oldest archive segment index.
+ if (oldestArchiveSegmentIdx < desc.idx)
+ oldestArchiveSegmentIdx = desc.idx;
+ }
+ }
+
+ return deleted;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int walArchiveSegments() {
+ long oldest = oldestArchiveSegmentIdx;
+
+ long lastArchived = archiver.lastArchivedAbsoluteIndex();
+
+ if (lastArchived == -1)
+ return 0;
+
+ int res = (int)(lastArchived - oldest);
+
+ return res >= 0 ? res : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserved(WALPointer ptr) {
+ FileWALPointer fPtr = (FileWALPointer)ptr;
+
+ FileArchiver archiver0 = archiver;
+
+ return archiver0 != null && archiver0.reserved(fPtr.index());
+ }
+
+ /**
+ * Lists files in archive directory and returns the index of last archived file.
+ *
+ * @return The absolute index of last archived file.
+ */
+ private long lastArchivedIndex() {
+ long lastIdx = -1;
+
+ for (File file : walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER)) {
+ try {
+ long idx = Long.parseLong(file.getName().substring(0, 16));
+
+ lastIdx = Math.max(lastIdx, idx);
+ }
+ catch (NumberFormatException | IndexOutOfBoundsException ignore) {
+
+ }
+ }
+
+ return lastIdx;
+ }
+
+ /**
+ * Lists files in archive directory and returns the index of last archived file.
+ *
+ * @return The absolute index of last archived file.
+ */
+ private IgniteBiTuple<Long, Long> scanMinMaxArchiveIndices() {
+ long minIdx = Integer.MAX_VALUE;
+ long maxIdx = -1;
+
+ for (File file : walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER)) {
+ try {
+ long idx = Long.parseLong(file.getName().substring(0, 16));
+
+ minIdx = Math.min(minIdx, idx);
+ maxIdx = Math.max(maxIdx, idx);
+ }
+ catch (NumberFormatException | IndexOutOfBoundsException ignore) {
+
+ }
+ }
+
+ return maxIdx == -1 ? null : F.t(minIdx, maxIdx);
+ }
+
+ /**
+ * Creates a directory specified by the given arguments.
+ *
+ * @param cfg Configured directory path, may be {@code null}.
+ * @param defDir Default directory path, will be used if cfg is {@code null}.
+ * @param consId Local node consistent ID.
+ * @param msg File description to print out on successful initialization.
+ * @return Initialized directory.
+ * @throws IgniteCheckedException If failed to initialize directory.
+ */
+ private File initDirectory(String cfg, String defDir, String consId, String msg) throws IgniteCheckedException {
+ File dir;
+
+ if (cfg != null) {
+ File workDir0 = new File(cfg);
+
+ dir = workDir0.isAbsolute() ?
+ new File(workDir0, consId) :
+ new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), cfg, false), consId);
+ }
+ else
+ dir = new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), defDir, false), consId);
+
+ U.ensureDirectory(dir, msg, log);
+
+ return dir;
+ }
+
+ /**
+ * @return Current log segment handle.
+ */
+ private FileWriteHandle currentHandle() {
+ return currentHnd;
+ }
+
+ /**
+ * @param cur Handle that failed to fit the given entry.
+ * @return Handle that will fit the entry.
+ */
+ private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
+ FileWriteHandle hnd = currentHandle();
+
+ if (hnd != cur)
+ return hnd;
+
+ if (hnd.close(true)) {
+ FileWriteHandle next = initNextWriteHandle(cur.idx);
+
+ boolean swapped = currentHndUpd.compareAndSet(this, hnd, next);
+
+ assert swapped : "Concurrent updates on rollover are not allowed";
+
+ // Let other threads to proceed with new segment.
+ hnd.signalNextAvailable();
+ }
+ else
+ hnd.awaitNext();
+
+ return currentHandle();
+ }
+
+ /**
+ * @param lastReadPtr Last read WAL file pointer.
+ * @return Initialized file write handle.
+ * @throws IgniteCheckedException If failed to initialize WAL write handle.
+ */
+ private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
+ long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
+
+ long segNo = absIdx % psCfg.getWalSegments();
+
+ File curFile = new File(walWorkDir, FileDescriptor.fileName(segNo));
+
+ int offset = lastReadPtr == null ? 0 : lastReadPtr.fileOffset();
+ int len = lastReadPtr == null ? 0 : lastReadPtr.length();
+
+ try {
+ RandomAccessFile file = new RandomAccessFile(curFile, "rw");
+
+ try {
+ // readSerializerVersion will change the channel position.
+ // This is fine because the FileWriteHandle consitructor will move it
+ // to offset + len anyways.
+ int serVer = readSerializerVersion(file, curFile, absIdx);
+
+ RecordSerializer ser = forVersion(cctx, serVer);
+
+ if (log.isInfoEnabled())
+ log.info("Resuming logging to WAL segment [file=" + curFile.getAbsolutePath() +
+ ", offset=" + offset + ", ver=" + serVer + ']');
+
+ FileWriteHandle hnd = new FileWriteHandle(
+ file,
+ absIdx,
+ cctx.igniteInstanceName(),
+ offset + len,
+ maxWalSegmentSize,
+ ser);
+
+ if (lastReadPtr == null) {
+ HeaderRecord header = new HeaderRecord(serializer.version());
+
+ header.size(serializer.size(header));
+
+ hnd.addRecord(header);
+ }
+
+ archiver.currentWalIndex(absIdx);
+
+ return hnd;
+ }
+ catch (IgniteCheckedException | IOException e) {
+ file.close();
+
+ throw e;
+ }
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * Fills the file header for a new segment.
+ * Calling this method signals we are done with the segment and it can be archived.
+ * If we don't have prepared file yet and achiever is busy this method blocks
+ *
+ * @param curIdx current segment released by WAL writer
+ * @return Initialized file handle.
+ * @throws StorageException If IO exception occurred.
+ * @throws IgniteCheckedException If failed.
+ */
+ private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteCheckedException {
+ try {
+ File nextFile = pollNextFile(curIdx);
+
+ if (log.isDebugEnabled())
+ log.debug("Switching to a new WAL segment: " + nextFile.getAbsolutePath());
+
+ RandomAccessFile file = new RandomAccessFile(nextFile, "rw");
+
+ FileWriteHandle hnd = new FileWriteHandle(
+ file,
+ curIdx + 1,
+ cctx.igniteInstanceName(),
+ 0,
+ maxWalSegmentSize,
+ serializer);
+
+ HeaderRecord header = new HeaderRecord(serializer.version());
+
+ header.size(serializer.size(header));
+
+ hnd.addRecord(header);
+
+ return hnd;
+ }
+ catch (IOException e) {
+ throw new StorageException(e);
+ }
+ }
+
+ /**
+ * Deletes temp files, creates and prepares new; Creates first segment if necessary
+ */
+ private void checkOrPrepareFiles() throws IgniteCheckedException {
+ // Clean temp files.
+ {
+ File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
+
+ if (!F.isEmpty(tmpFiles)) {
+ for (File tmp : tmpFiles) {
+ boolean deleted = tmp.delete();
+
+ if (!deleted)
+ throw new IgniteCheckedException("Failed to delete previously created temp file " +
+ "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
+ }
+ }
+ }
+
+ File[] allFiles = walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER);
+
+ if (allFiles.length != 0 && allFiles.length > psCfg.getWalSegments())
+ throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
+ "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + psCfg.getWalSegments() + ']');
+
+ // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
+ if (allFiles.length == 0) {
+ File first = new File(walWorkDir, FileDescriptor.fileName(0));
+
+ createFile(first);
+ }
+ else
+ checkFiles(0, false, null);
+ }
+
+ /**
+ * Clears the file with zeros.
+ *
+ * @param file File to format.
+ */
+ private void formatFile(File file) throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
+
+ try (RandomAccessFile rnd = new RandomAccessFile(file, "rw")) {
+ int left = psCfg.getWalSegmentSize();
+
+ if (mode == WALMode.DEFAULT) {
+ while (left > 0) {
+ int toWrite = Math.min(FILL_BUF.length, left);
+
+ rnd.write(FILL_BUF, 0, toWrite);
+
+ left -= toWrite;
+ }
+
+ rnd.getChannel().force(false);
+ }
+ else
+ rnd.setLength(0);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * Creates a file atomically with temp file.
+ *
+ * @param file File to create.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void createFile(File file) throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
+
+ File tmp = new File(file.getParent(), file.getName() + ".tmp");
+
+ formatFile(tmp);
+
+ try {
+ Files.move(tmp.toPath(), file.toPath());
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
+ file.getAbsolutePath(), e);
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Created WAL segment [file=" + file.getAbsolutePath() + ", size=" + file.length() + ']');
+ }
+
+ /**
+ * Retrieves next available file to write WAL data, waiting
+ * if necessary for a segment to become available.
+ *
+ * @param curIdx Current absolute WAL segment index.
+ * @return File ready for use as new WAL segment.
+ * @throws IgniteCheckedException If failed.
+ */
+ private File pollNextFile(long curIdx) throws IgniteCheckedException {
+ // Signal to archiver that we are done with the segment and it can be archived.
+ long absNextIdx = archiver.nextAbsoluteSegmentIndex(curIdx);
+
+ long segmentIdx = absNextIdx % psCfg.getWalSegments();
+
+ return new File(walWorkDir, FileDescriptor.fileName(segmentIdx));
+ }
+
+ /**
+ * @param ver Serializer version.
+ * @return Entry serializer.
+ */
+ private static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws IgniteCheckedException {
+ if (ver <= 0)
+ throw new IgniteCheckedException("Failed to create a serializer (corrupted WAL file).");
+
+ switch (ver) {
+ case 1:
+ return new RecordV1Serializer(cctx);
+
+ default:
+ throw new IgniteCheckedException("Failed to create a serializer with the given version " +
+ "(forward compatibility is not supported): " + ver);
+ }
+ }
+
+ /**
+ * @return Sorted WAL files descriptors.
+ */
+ private static FileDescriptor[] scan(File[] allFiles) {
+ if (allFiles == null)
+ return EMPTY_DESCRIPTORS;
+
+ FileDescriptor[] descs = new FileDescriptor[allFiles.length];
+
+ for (int i = 0; i < allFiles.length; i++) {
+ File f = allFiles[i];
+
+ descs[i] = new FileDescriptor(f);
+ }
+
+ Arrays.sort(descs);
+
+ return descs;
+ }
+
+ /**
+ * File archiver operates on absolute segment indexes. For any given absolute segment index N we can calculate
+ * the work WAL segment: S(N) = N % psCfg.walSegments.
+ * When a work segment is finished, it is given to the archiver. If the absolute index of last archived segment
+ * is denoted by A and the absolute index of next segment we want to write is denoted by W, then we can allow
+ * write to S(W) if W - A <= walSegments. <br>
+ *
+ * Monitor of current object is used for notify on:
+ * <ul>
+ * <li>exception occurred ({@link FileArchiver#cleanException}!=null)</li>
+ * <li>stopping thread ({@link FileArchiver#stopped}==true)</li>
+ * <li>current file index changed ({@link FileArchiver#curAbsWalIdx})</li>
+ * <li>last archived file index was changed ({@link FileArchiver#lastAbsArchivedIdx})</li>
+ * <li>some WAL index was removed from {@link FileArchiver#locked} map</li>
+ * </ul>
+ */
+ private class FileArchiver extends Thread {
+ /** Exception which occurred during initial creation of files or during archiving WAL segment */
+ private IgniteCheckedException cleanException;
+
+ /**
+ * Absolute current segment index WAL Manger writes to. Guarded by <code>this</code>.
+ * Incremented during rollover. Also may be directly set if WAL is resuming logging after start.
+ */
+ private long curAbsWalIdx = -1;
+
+ /** Last archived file index (absolute, 0-based). Guarded by <code>this</code>. */
+ private long lastAbsArchivedIdx = -1;
+
+ /** current thread stopping advice */
+ private volatile boolean stopped;
+
+ /** */
+ private NavigableMap<Long, Integer> reserved = new TreeMap<>();
+
+ /**
+ * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may
+ * come from {@link RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
+ */
+ private Map<Long, Integer> locked = new HashMap<>();
+
+ /**
+ *
+ */
+ private FileArchiver(long lastAbsArchivedIdx) {
+ super("wal-file-archiver%" + cctx.igniteInstanceName());
+
+ this.lastAbsArchivedIdx = lastAbsArchivedIdx;
+ }
+
+ /**
+ * @return Last archived segment absolute index.
+ */
+ private synchronized long lastArchivedAbsoluteIndex() {
+ return lastAbsArchivedIdx;
+ }
+
+ /**
+ * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
+ */
+ private void shutdown() throws IgniteInterruptedCheckedException {
+ synchronized (this) {
+ stopped = true;
+
+ notifyAll();
+ }
+
+ U.join(this);
+ }
+
+ /**
+ * @param curAbsWalIdx Current absolute WAL segment index.
+ */
+ private void currentWalIndex(long curAbsWalIdx) {
+ synchronized (this) {
+ this.curAbsWalIdx = curAbsWalIdx;
+
+ notifyAll();
+ }
+ }
+
+ /**
+ * @param absIdx Index for reservation.
+ */
+ private synchronized void reserve(long absIdx) {
+ Integer cur = reserved.get(absIdx);
+
+ if (cur == null)
+ reserved.put(absIdx, 1);
+ else
+ reserved.put(absIdx, cur + 1);
+ }
+
+ /**
+ * @param absIdx Index for reservation.
+ * @return {@code True} if index is reserved.
+ */
+ private synchronized boolean reserved(long absIdx) {
+ return locked.containsKey(absIdx) || reserved.floorKey(absIdx) != null;
+ }
+
+ /**
+ * @param absIdx Reserved index.
+ */
+ private synchronized void release(long absIdx) {
+ Integer cur = reserved.get(absIdx);
+
+ assert cur != null && cur >= 1 : cur;
+
+ if (cur == 1)
+ reserved.remove(absIdx);
+ else
+ reserved.put(absIdx, cur - 1);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ allocateRemainingFiles();
+ }
+ catch (IgniteCheckedException e) {
+ synchronized (this) {
+ // Stop the thread and report to starter.
+ cleanException = e;
+
+ notifyAll();
+
+ return;
+ }
+ }
+
+ try {
+ synchronized (this) {
+ while (curAbsWalIdx == -1 && !stopped)
+ wait();
+
+ if (curAbsWalIdx != 0 && lastAbsArchivedIdx == -1)
+ lastAbsArchivedIdx = curAbsWalIdx - 1;
+ }
+
+ while (!Thread.currentThread().isInterrupted() && !stopped) {
+ long toArchive;
+
+ synchronized (this) {
+ assert lastAbsArchivedIdx <= curAbsWalIdx : "lastArchived=" + lastAbsArchivedIdx +
+ ", current=" + curAbsWalIdx;
+
+ while (lastAbsArchivedIdx >= curAbsWalIdx - 1 && !stopped)
+ wait();
+
+ toArchive = lastAbsArchivedIdx + 1;
+ }
+
+ if (stopped)
+ break;
+
+ try {
+ File workFile = archiveSegment(toArchive);
+
+ synchronized (this) {
+ while (locked.containsKey(toArchive) && !stopped)
+ wait();
+
+ // Firstly, format working file
+ if (!stopped)
+ formatFile(workFile);
+
+ // Then increase counter to allow rollover on clean working file
+ lastAbsArchivedIdx = toArchive;
+
+ notifyAll();
+ }
+ }
+ catch (IgniteCheckedException e) {
+ synchronized (this) {
+ cleanException = e;
+
+ notifyAll();
+ }
+ }
+ }
+ }
+ catch (InterruptedException ignore) {
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Gets the absolute index of the next WAL segment available to write.
+ * Blocks till there are available file to write
+ *
+ * @param curIdx Current absolute index that we want to increment.
+ * @return Next index (curIdx+1) when it is ready to be written.
+ * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
+ */
+ private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
+ try {
+ synchronized (this) {
+ if (cleanException != null)
+ throw cleanException;
+
+ assert curIdx == curAbsWalIdx;
+
+ curAbsWalIdx++;
+
+ // Notify archiver thread.
+ notifyAll();
+
+ while (curAbsWalIdx - lastAbsArchivedIdx > psCfg.getWalSegments() && cleanException == null)
+ wait();
+
+ return curAbsWalIdx;
+ }
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IgniteInterruptedCheckedException(e);
+ }
+ }
+
+ /**
+ * @param absIdx Segment absolute index.
+ * @return {@code True} if can read, {@code false} if work segment
+ */
+ @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
+ private boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) {
+ synchronized (this) {
+ if (lastAbsArchivedIdx >= absIdx)
+ return true;
+
+ Integer cur = locked.get(absIdx);
+
+ cur = cur == null ? 1 : cur + 1;
+
+ locked.put(absIdx, cur);
+
+ if (log.isDebugEnabled())
+ log.debug("Reserved work segment [absIdx=" + absIdx + ", pins=" + cur + ']');
+
+ return false;
+ }
+ }
+
+ /**
+ * @param absIdx Segment absolute index.
+ */
+ @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
+ private void releaseWorkSegment(long absIdx) {
+ synchronized (this) {
+ Integer cur = locked.get(absIdx);
+
+ assert cur != null && cur > 0;
+
+ if (cur == 1) {
+ locked.remove(absIdx);
+
+ if (log.isDebugEnabled())
+ log.debug("Fully released work segment (ready to archive) [absIdx=" + absIdx + ']');
+ }
+ else {
+ locked.put(absIdx, cur - 1);
+
+ if (log.isDebugEnabled())
+ log.debug("Partially released work segment [absIdx=" + absIdx + ", pins=" + (cur - 1) + ']');
+ }
+
+ notifyAll();
+ }
+ }
+
+ /**
+ * @param absIdx Absolute index to archive.
+ */
+ private File archiveSegment(long absIdx) throws IgniteCheckedException {
+ long segIdx = absIdx % psCfg.getWalSegments();
+
+ File origFile = new File(walWorkDir, FileDescriptor.fileName(segIdx));
+
+ String name = FileDescriptor.fileName(absIdx);
+
+ File dstTmpFile = new File(walArchiveDir, name + ".tmp");
+
+ File dstFile = new File(walArchiveDir, name);
+
+ if (log.isDebugEnabled())
+ log.debug("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
+ ", origFile=" + origFile.getAbsolutePath() + ", dstFile=" + dstFile.getAbsolutePath() + ']');
+
+ try {
+ Files.deleteIfExists(dstTmpFile.toPath());
+
+ Files.copy(origFile.toPath(), dstTmpFile.toPath());
+
+ Files.move(dstTmpFile.toPath(), dstFile.toPath());
+
+ if (mode == WALMode.DEFAULT) {
+ try (RandomAccessFile f0 = new RandomAccessFile(dstFile, "rw")) {
+ f0.getChannel().force(false);
+ }
+ }
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to archive WAL segment [" +
+ "srcFile=" + origFile.getAbsolutePath() +
+ ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Copied file [src=" + origFile.getAbsolutePath() +
+ ", dst=" + dstFile.getAbsolutePath() + ']');
+
+ return origFile;
+ }
+
+ /**
+ *
+ */
+ private boolean checkStop() {
+ return stopped;
+ }
+
+ /**
+ * Background creation of all segments except first. First segment was created in main thread by
+ * {@link FileWriteAheadLogManager#checkOrPrepareFiles()}
+ */
+ private void allocateRemainingFiles() throws IgniteCheckedException {
+ checkFiles(1, true, new IgnitePredicate<Integer>() {
+ @Override public boolean apply(Integer integer) {
+ return !checkStop();
+ }
+ });
+ }
+ }
+
+ /**
+ * Validate files depending on {@link PersistentStoreConfiguration#getWalSegments()} and create if need.
+ * Check end when exit condition return false or all files are passed.
+ *
+ * @param startWith Start with.
+ * @param create Flag create file.
+ * @param p Predicate Exit condition.
+ * @throws IgniteCheckedException if validation or create file fail.
+ */
+ private void checkFiles(int startWith, boolean create, IgnitePredicate<Integer> p) throws IgniteCheckedException {
+ for (int i = startWith; i < psCfg.getWalSegments() && (p == null || (p != null && p.apply(i))); i++) {
+ File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
+
+ if (checkFile.exists()) {
+ if (checkFile.isDirectory())
+ throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
+ "the same name already exists): " + checkFile.getAbsolutePath());
+ else if (checkFile.length() != psCfg.getWalSegmentSize() && mode == WALMode.DEFAULT)
+ throw new IgniteCheckedException("Failed to initialize WAL log segment " +
+ "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
+ }
+ else if (create)
+ createFile(checkFile);
+ }
+ }
+
+ /**
+ * @param rf Random access file.
+ * @param file File object.
+ * @param idx File index to read.
+ * @return Serializer version stored in the file.
+ * @throws IOException If failed to read serializer version.
+ * @throws IgniteCheckedException If failed to read serializer version.
+ */
+ private int readSerializerVersion(RandomAccessFile rf, File file, long idx)
+ throws IOException, IgniteCheckedException {
+ try {
+ ByteBuffer buf = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ FileInput in = new FileInput(rf.getChannel(), buf);
+
+ // Header record must be agnostic to the serializer version.
+ WALRecord rec = serializer.readRecord(in, new FileWALPointer(idx, 0, 0));
+
+ if (rec.type() != WALRecord.RecordType.HEADER_RECORD)
+ throw new IOException("Missing file header record: " + file.getAbsoluteFile());
+
+ return ((HeaderRecord)rec).version();
+ }
+ catch (SegmentEofException | EOFException ignore) {
+ return serializer.version();
+ }
+ }
+
+ /**
+ * WAL file descriptor.
+ */
+ private static class FileDescriptor implements Comparable<FileDescriptor> {
+ /** */
+ protected final File file;
+
+ /** Absolute WAL segment file index */
+ protected final long idx;
+
+ /**
+ * @param file File.
+ */
+ private FileDescriptor(File file) {
+ this(file, null);
+ }
+
+ /**
+ * @param file File.
+ * @param idx Absolute WAL segment file index.
+ */
+ private FileDescriptor(File file, Long idx) {
+ this.file = file;
+
+ String fileName = file.getName();
+
+ assert fileName.endsWith(WAL_SEGMENT_FILE_EXT);
+
+ int end = fileName.length() - WAL_SEGMENT_FILE_EXT.length();
+
+ this.idx = idx == null ? Long.parseLong(fileName.substring(0, end)) : idx;
+ }
+
+ /**
+ * @param segment Segment index.
+ * @return Segment file name.
+ */
+ private static String fileName(long segment) {
+ SB b = new SB();
+
+ String segmentStr = Long.toString(segment);
+
+ for (int i = segmentStr.length(); i < 16; i++)
+ b.a('0');
+
+ b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
+
+ return b.toString();
+ }
+
+ /**
+ * @param segment Segment number as integer.
+ * @return Segment number as aligned string.
+ */
+ private static String segmentNumber(long segment) {
+ SB b = new SB();
+
+ String segmentStr = Long.toString(segment);
+
+ for (int i = segmentStr.length(); i < 16; i++)
+ b.a('0');
+
+ b.a(segmentStr);
+
+ return b.toString();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(FileDescriptor o) {
+ return Long.compare(idx, o.idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof FileDescriptor))
+ return false;
+
+ FileDescriptor that = (FileDescriptor)o;
+
+ return idx == that.idx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return (int)(idx ^ (idx >>> 32));
+ }
+ }
+
+ /**
+ *
+ */
+ private abstract static class FileHandle {
+ /** */
+ protected RandomAccessFile file;
+
+ /** */
+ protected FileChannel ch;
+
+ /** */
+ protected final long idx;
+
+ /** */
+ protected String gridName;
+
+ /**
+ * @param file File.
+ * @param idx Index.
+ */
+ private FileHandle(RandomAccessFile file, long idx, String gridName) {
+ this.file = file;
+ this.idx = idx;
+ this.gridName = gridName;
+
+ ch = file.getChannel();
+
+ assert ch != null;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class ReadFileHandle extends FileHandle {
+ /** Entry serializer. */
+ private RecordSerializer ser;
+
+ /** */
+ private FileInput in;
+
+ /** */
+ private boolean workDir;
+
+ /**
+ * @param file File to read.
+ * @param idx File index.
+ * @param ser Entry serializer.
+ * @param in File input.
+ */
+ private ReadFileHandle(
+ RandomAccessFile file,
+ long idx,
+ String gridName,
+ RecordSerializer ser,
+ FileInput in
+ ) {
+ super(file, idx, gridName);
+
+ this.ser = ser;
+ this.in = in;
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed to close the WAL segment file.
+ */
+ public void close() throws IgniteCheckedException {
+ try {
+ file.close();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+ }
+
+ /**
+ * File handle for one log segment.
+ */
+ @SuppressWarnings("SignalWithoutCorrespondingAwait")
+ private class FileWriteHandle extends FileHandle {
+ /** */
+ private final RecordSerializer serializer;
+
+ /** See {@link FileWriteAheadLogManager#maxWalSegmentSize} */
+ private final long maxSegmentSize;
+
+ /**
+ * Accumulated WAL records chain.
+ * This reference points to latest WAL record.
+ * When writing records chain is iterated from latest to oldest (see {@link WALRecord#previous()})
+ * Records from chain are saved into buffer in reverse order
+ */
+ private final AtomicReference<WALRecord> head = new AtomicReference<>();
+
+ /** Position in current file after the end of last written record (incremented after file channel write operation) */
+ private volatile long written;
+
+ /** */
+ private volatile long lastFsyncPos;
+
+ /** Environment failure. */
+ private volatile Throwable envFailed;
+
+ /** Stop guard to provide warranty that only one thread will be successful in calling {@link #close(boolean)}*/
+ private final AtomicBoolean stop = new AtomicBoolean(false);
+
+ /** */
+ private final Lock lock = new ReentrantLock();
+
+ /** Condition activated each time writeBuffer() completes. Used to wait previously flushed write to complete */
+ private final Condition writeComplete = lock.newCondition();
+
+ /** Condition for timed wait of several threads, see {@link PersistentStoreConfiguration#getWalFsyncDelay()} */
+ private final Condition fsync = lock.newCondition();
+
+ /**
+ * Next segment available condition.
+ * Protection from "spurious wakeup" is provided by predicate {@link #ch}=<code>null</code>
+ */
+ private final Condition nextSegment = lock.newCondition();
+
+ /**
+ * @param file Mapped file to use.
+ * @param idx Absolute WAL segment file index for easy access.
+ * @param pos Position.
+ * @param maxSegmentSize Max segment size.
+ * @param serializer Serializer.
+ * @throws IOException If failed.
+ */
+ private FileWriteHandle(
+ RandomAccessFile file,
+ long idx,
+ String gridName,
+ long pos,
+ long maxSegmentSize,
+ RecordSerializer serializer
+ ) throws IOException {
+ super(file, idx, gridName);
+
+ assert serializer != null;
+
+ ch.position(pos);
+
+ this.maxSegmentSize = maxSegmentSize;
+ this.serializer = serializer;
+
+ head.set(new FakeRecord(new FileWALPointer(idx, (int)pos, 0)));
+ written = pos;
+ lastFsyncPos = pos;
+ }
+
+ /**
+ * @param rec Record to be added to record chain as new {@link #head}
+ * @return Pointer or null if roll over to next segment is required or already started by other thread.
+ * @throws StorageException If failed.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException, IgniteCheckedException {
+ assert rec.size() > 0 || rec.getClass() == FakeRecord.class;
+
+ boolean flushed = false;
+
+ for (; ; ) {
+ WALRecord h = head.get();
+
+ long nextPos = nextPosition(h);
+
+ // It is important that we read `stop` after `head` in this loop for correct close,
+ // because otherwise we will have a race on the last flush in close.
+ if (nextPos + rec.size() >= maxSegmentSize || stop.get()) {
+ // Can not write to this segment, need to switch to the next one.
+ return null;
+ }
+
+ int newChainSize = h.chainSize() + rec.size();
+
+ if (newChainSize > tlbSize && !flushed) {
+ boolean res = h.previous() == null || flush(h);
+
+ if (rec.size() > tlbSize)
+ flushed = res;
+
+ continue;
+ }
+
+ rec.chainSize(newChainSize);
+ rec.previous(h);
+
+ FileWALPointer ptr = new FileWALPointer(
+ idx,
+ (int)nextPos,
+ rec.size(),
+ // We need to force checkpoint records into file in BACKGROUND WALMode.
+ mode == WALMode.BACKGROUND && rec instanceof CheckpointRecord);
+
+ rec.position(ptr);
+
+ if (head.compareAndSet(h, rec))
+ return ptr;
+ }
+ }
+
+ /**
+ * @param rec Record.
+ * @return Position for the next record.
+ */
+ private long nextPosition(WALRecord rec) {
+ return recordOffset(rec) + rec.size();
+ }
+
+ /**
+ * Flush or wait for concurrent flush completion.
+ *
+ * @param ptr Pointer.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
+ long expWritten;
+
+ if (ptr != null) {
+ // If requested obsolete file index, it must be already flushed by close.
+ if (ptr.index() != idx)
+ return;
+
+ expWritten = ptr.fileOffset();
+ }
+ else // We read head position before the flush because otherwise we can get wrong position.
+ expWritten = recordOffset(head.get());
+
+ if (flush(ptr))
+ return;
+
+ // Spin-wait for a while before acquiring the lock.
+ for (int i = 0; i < 64; i++) {
+ if (written >= expWritten)
+ return;
+ }
+
+ // If we did not flush ourselves then await for concurrent flush to complete.
+ lock.lock();
+
+ try {
+ while (written < expWritten && envFailed == null)
+ U.await(writeComplete);
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @param ptr Pointer.
+ * @return {@code true} If the flush really happened.
+ * @throws IgniteCheckedException If failed.
+ * @throws StorageException If failed.
+ */
+ private boolean flush(FileWALPointer ptr) throws IgniteCheckedException, StorageException {
+ if (ptr == null) { // Unconditional flush.
+ for (; ; ) {
+ WALRecord expHead = head.get();
+
+ if (expHead.previous() == null) {
+ assert expHead instanceof FakeRecord;
+
+ return false;
+ }
+
+ if (flush(expHead))
+ return true;
+ }
+ }
+
+ assert ptr.index() == idx;
+
+ for (; ; ) {
+ WALRecord h = head.get();
+
+ // If current chain begin position is greater than requested, then someone else flushed our changes.
+ if (chainBeginPosition(h) > ptr.fileOffset())
+ return false;
+
+ if (flush(h))
+ return true; // We are lucky.
+ }
+ }
+
+ /**
+ * @param h Head of the chain.
+ * @return Chain begin position.
+ */
+ private long chainBeginPosition(WALRecord h) {
+ return recordOffset(h) + h.size() - h.chainSize();
+ }
+
+ /**
+ * @param expHead Expected head of chain. If head was changed, flush is not performed in this thread
+ * @throws IgniteCheckedException If failed.
+ * @throws StorageException If failed.
+ */
+ private boolean flush(WALRecord expHead) throws StorageException, IgniteCheckedException {
+ if (expHead.previous() == null) {
+ assert expHead instanceof FakeRecord;
+
+ return false;
+ }
+
+ // Fail-fast before CAS.
+ checkEnvironment();
+
+ if (!head.compareAndSet(expHead, new FakeRecord(new FileWALPointer(idx, (int)nextPosition(expHead), 0))))
+ return false;
+
+ // At this point we grabbed the piece of WAL chain.
+ // Any failure in this code must invalidate the environment.
+ try {
+ // We can safely allow other threads to start building next chains while we are doing flush here.
+ ByteBuffer buf;
+
+ boolean tmpBuf = false;
+
+ if (expHead.chainSize() > tlbSize) {
+ buf = GridUnsafe.allocateBuffer(expHead.chainSize());
+
+ tmpBuf = true; // We need to manually release this temporary direct buffer.
+ }
+ else
+ buf = tlb.get();
+
+ try {
+ long pos = fillBuffer(buf, expHead);
+
+ writeBuffer(pos, buf);
+ }
+ finally {
+ if (tmpBuf)
+ GridUnsafe.freeBuffer(buf);
+ }
+
+ return true;
+ }
+ catch (Throwable e) {
+ invalidateEnvironment(e);
+
+ throw e;
+ }
+ }
+
+ /**
+ * Serializes WAL records chain to provided byte buffer
+ * @param buf Buffer, will be filled with records chain from end to beginning
+ * @param head Head of the chain to write to the buffer.
+ * @return Position in file for this buffer.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long fillBuffer(ByteBuffer buf, WALRecord head) throws IgniteCheckedException {
+ final int limit = head.chainSize();
+
+ assert limit <= buf.capacity();
+
+ buf.rewind();
+ buf.limit(limit);
+
+ do {
+ buf.position(head.chainSize() - head.size());
+ buf.limit(head.chainSize()); // Just to make sure that serializer works in bounds.
+
+ try {
+ serializer.writeRecord(head, buf);
+ }
+ catch (RuntimeException e) {
+ throw new IllegalStateException("Failed to write record: " + head, e);
+ }
+
+ assert !buf.hasRemaining() : "Reported record size is greater than actual: " + head;
+
+ head = head.previous();
+ }
+ while (head.previous() != null);
+
+ assert head instanceof FakeRecord : head.getClass();
+
+ buf.rewind();
+ buf.limit(limit);
+
+ return recordOffset(head);
+ }
+
+ /**
+ * Non-blocking check if this pointer needs to be sync'ed.
+ *
+ * @param ptr WAL pointer to check.
+ * @return {@code False} if this pointer has been already sync'ed.
+ */
+ private boolean needFsync(FileWALPointer ptr) {
+ // If index has changed, it means that the log was rolled over and already sync'ed.
+ // If requested position is smaller than last sync'ed, it also means all is good.
+ // If position is equal, then our record is the last not synced.
+ return idx == ptr.index() && lastFsyncPos <= ptr.fileOffset();
+ }
+
+ /**
+ * @return Pointer to the end of the last written record (probably not fsync-ed).
+ */
+ private FileWALPointer position() {
+ lock.lock();
+
+ try {
+ return new FileWALPointer(idx, (int)written, 0);
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @param ptr Pointer to sync.
+ * @throws StorageException If failed.
+ */
+ private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedException {
+ lock.lock();
+
+ try {
+ if (ptr != null) {
+ if (!needFsync(ptr))
+ return;
+
+ if (fsyncDelay > 0 && !stop.get()) {
+ // Delay fsync to collect as many updates as possible: trade latency for throughput.
+ U.await(fsync, fsyncDelay, TimeUnit.NANOSECONDS);
+
+ if (!needFsync(ptr))
+ return;
+ }
+ }
+
+ flushOrWait(ptr);
+
+ if (lastFsyncPos != written) {
+ assert lastFsyncPos < written; // Fsync position must be behind.
+
+ boolean metricsEnabled = metrics.metricsEnabled();
+
+ long start = metricsEnabled ? System.nanoTime() : 0;
+
+ try {
+ ch.force(false);
+ }
+ catch (IOException e) {
+ throw new StorageException(e);
+ }
+
+ lastFsyncPos = written;
+
+ if (fsyncDelay > 0)
+ fsync.signalAll();
+
+ long end = metricsEnabled ? System.nanoTime() : 0;
+
+ if (metricsEnabled)
+ metrics.onFsync(end - start);
+ }
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @return {@code true} If this thread actually closed the segment.
+ * @throws IgniteCheckedException If failed.
+ * @throws StorageException If failed.
+ */
+ private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
+ if (stop.compareAndSet(false, true)) {
+ // Here we can be sure that no other records will be added and this fsync will be the last.
+ if (mode == WALMode.DEFAULT)
+ fsync(null);
+ else
+ flushOrWait(null);
+
+ try {
+ if (rollOver && written < (maxSegmentSize - 1)) {
+ ByteBuffer allocate = ByteBuffer.allocate(1);
+ allocate.put((byte) WALRecord.RecordType.SWITCH_SEGMENT_RECORD.ordinal());
+
+ ch.write(allocate, written);
+
+ if (mode == WALMode.DEFAULT)
+ ch.force(false);
+ }
+
+ ch.close();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Closed WAL write handle [idx=" + idx + "]");
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * Signals next segment available to wake up other worker threads waiting for WAL to write
+ */
+ private void signalNextAvailable() {
+ lock.lock();
+
+ try {
+ assert head.get() instanceof FakeRecord: "head";
+ assert written == lastFsyncPos || mode != WALMode.DEFAULT :
+ "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ']';
+
+ ch = null;
+
+ nextSegment.signalAll();
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void awaitNext() throws IgniteCheckedException {
+ lock.lock();
+
+ try {
+ while (ch != null)
+ U.await(nextSegment);
+ }
+ finally {
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @param pos Position in file to start write from.
+ * May be checked against actual position to wait previous writes to complete
+ * @param buf Buffer to write to file
+ * @throws StorageException If failed.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("TooBroadScope")
+ private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, IgniteCheckedException {
+ boolean interrupted = false;
+
+ lock.lock();
+
+ try {
+ assert ch != null : "Writing to a closed segment.";
+
+ checkEnvironment();
+
+ long lastLogged = U.currentTimeMillis();
+
+ long logBackoff = 2_000;
+
+ // If we were too fast, need to wait previous writes to complete.
+ while (written != pos) {
+ assert written < pos : "written = " + written + ", pos = " + pos; // No one can write further than we are now.
+
+ // Permutation occurred between blocks write operations.
+ // Order of acquiring lock is not the same as order of write.
+ long now = U.currentTimeMillis();
+
+ if (now - lastLogged >= logBackoff) {
+ if (logBackoff < 60 * 60_000)
+ logBackoff *= 2;
+
+ U.warn(log, "Still waiting for a concurrent write to complete [written=" + written +
+ ", pos=" + pos + ", lastFsyncPos=" + lastFsyncPos + ", stop=" + stop.get() +
+ ", actualPos=" + safePosition() + ']');
+
+ lastLogged = now;
+ }
+
+ try {
+ writeComplete.await(2, TimeUnit.SECONDS);
+ }
+ catch (InterruptedException ignore) {
+ interrupted = true;
+ }
+
+ checkEnvironment();
+ }
+
+ // Do the write.
+ int size = buf.remaining();
+
+ assert size > 0 : size;
+
+ try {
+ assert written == ch.position();
+
+ do {
+ ch.write(buf);
+ }
+ while (buf.hasRemaining());
+
+ written += size;
+
+ metrics.onWalBytesWritten(size);
+
+ assert written == ch.position();
+ }
+ catch (IOException e) {
+ invalidateEnvironmentLocked(e);
+
+ throw new StorageException(e);
+ }
+ }
+ finally {
+ writeComplete.signalAll();
+
+ lock.unlock();
+
+ if (interrupted)
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * @param e Exception to set as a cause for all further operations.
+ */
+ private void invalidateEnvironment(Throwable e) {
+ lock.lock();
+
+ try {
+ invalidateEnvironmentLocked(e);
+ }
+ finally {
+ writeComplete.signalAll();
+
+ lock.unlock();
+ }
+ }
+
+ /**
+ * @param e Exception to set as a cause for all further operations.
+ */
+ private void invalidateEnvironmentLocked(Throwable e) {
+ if (envFailed == null) {
+ envFailed = e;
+
+ U.error(log, "IO error encountered while running WAL flush. All further operations will be failed and " +
+ "local node will be stopped.", e);
+
+ new Thread() {
+ @Override public void run() {
+ G.stop(gridName, true);
+ }
+ }.start();
+ }
+ }
+
+ /**
+ * @throws StorageException If environment is no longer valid and we missed a WAL write.
+ */
+ private void checkEnvironment() throws StorageException {
+ if (envFailed != null)
+ throw new StorageException("Failed to flush WAL buffer (environment was invalidated by a " +
+ "previous error)", envFailed);
+ }
+
+ /**
+ * @return Safely reads current position of the file channel as String. Will return "null" if channel is null.
+ */
+ private String safePosition() {
+ FileChannel ch = this.ch;
+
+ if (ch == null)
+ return "null";
+
+ try {
+ return String.valueOf(ch.position());
+ }
+ catch (IOException e) {
+ return "{Failed to read channel position: " + e.getMessage() + "}";
+ }
+ }
+ }
+
+ /**
+ * Gets WAL record offset relative to the WAL segment file beginning.
+ *
+ * @param rec WAL record.
+ * @return File offset.
+ */
+ private static int recordOffset(WALRecord rec) {
+ FileWALPointer ptr = (FileWALPointer)rec.position();
+
+ assert ptr != null;
+
+ return ptr.fileOffset();
+ }
+
+ /**
+ * Fake record is zero-sized record, which is not stored into file.
+ * Fake record is used for storing position in file {@link WALRecord#position()}.
+ * Fake record is allowed to have no previous record.
+ */
+ private static final class FakeRecord extends WALRecord {
+ /**
+ * @param pos Position.
+ */
+ FakeRecord(FileWALPointer pos) {
+ position(pos);
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return null;
+ }
+ }
+
+ /**
+ * Iterator over WAL-log.
+ */
+ private static class RecordsIterator extends GridCloseableIteratorAdapter<IgniteBiTuple<WALPointer, WALRecord>>
+ implements WALIterator {
+ /** */
+ private static final long serialVersionUID = 0L;
+ /** */
+ private final File walWorkDir;
+
+ /** */
+ private final File walArchiveDir;
+
+ /** */
+ private final FileArchiver archiver;
+
+ /** */
+ private final PersistentStoreConfiguration psCfg;
+
+ /** */
+ private final RecordSerializer serializer;
+
+ /** */
+ private final GridCacheSharedContext cctx;
+
+ /** */
+ private FileWALPointer start;
+
+ /** */
+ private FileWALPointer end;
+
+ /** */
+ private IgniteBiTuple<WALPointer, WALRecord> curRec;
+
+ /** */
+ private long curIdx = -1;
+
+ /** */
+ private ReadFileHandle curHandle;
+
+ /** */
+ private ByteBuffer buf;
+
+ /** */
+ private IgniteLogger log;
+
+ /**
+ * @param cctx Shared context.
+ * @param walWorkDir WAL work dir.
+ * @param walArchiveDir WAL archive dir.
+ * @param start Optional start pointer.
+ * @param end Optional end pointer.
+ * @param psCfg Database configuration.
+ * @param serializer Serializer.
+ * @param archiver Archiver.
+ * @throws IgniteCheckedException If failed to initialize WAL segment.
+ */
+ private RecordsIterator(
+ GridCacheSharedContext cctx,
+ File walWorkDir,
+ File walArchiveDir,
+ FileWALPointer start,
+ FileWALPointer end,
+ PersistentStoreConfiguration psCfg,
+ RecordSerializer serializer,
+ FileArchiver archiver,
+ IgniteLogger log,
+ int tlbSize
+ ) throws IgniteCheckedException {
+ this.cctx = cctx;
+ this.walWorkDir = walWorkDir;
+ this.walArchiveDir = walArchiveDir;
+ this.psCfg = psCfg;
+ this.serializer = serializer;
+ this.archiver = archiver;
+ this.start = start;
+ this.end = end;
+ this.log = log;
+
+ int buffSize = Math.min(16 * tlbSize, psCfg.getWalRecordIteratorBufferSize());
+
+ // Do not allocate direct buffer for iterator.
+ buf = ByteBuffer.allocate(buffSize);
+ buf.order(ByteOrder.nativeOrder());
+
+ init();
+
+ advance();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteBiTuple<WALPointer, WALRecord> onNext() throws IgniteCheckedException {
+ IgniteBiTuple<WALPointer, WALRecord> ret = curRec;
+
+ advance();
+
+ return ret;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean onHasNext() throws IgniteCheckedException {
+ return curRec != null;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onClose() throws IgniteCheckedException {
+ curRec = null;
+
+ if (curHandle != null) {
+ curHandle.close();
+
+ if (curHandle.workDir)
+ releaseWorkSegment(curIdx);
+
+ curHandle = null;
+ }
+
+ curIdx = Integer.MAX_VALUE;
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed to initialize first file handle.
+ */
+ private void init() throws IgniteCheckedException {
+ FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER));
+
+ if (start != null) {
+ if (!F.isEmpty(descs)) {
+ if (descs[0].idx > start.index())
+ throw new IgniteCheckedException("WAL history is too short " +
+ "[descs=" + Arrays.asList(descs) + ", start=" + start + ']');
+
+ for (FileDescriptor desc : descs) {
+ if (desc.idx == start.index()) {
+ curIdx = start.index();
+
+ break;
+ }
+ }
+
+ if (curIdx == -1) {
+ long lastArchived = descs[descs.length - 1].idx;
+
+ if (lastArchived > start.index())
+ throw new IgniteCheckedException("WAL history is corrupted (segment is missing): " + start);
+
+ // This pointer may be in work files because archiver did not
+ // copy the file yet, check that it is not too far forward.
+ curIdx = start.index();
+ }
+ }
+ else {
+ // This means that whole checkpoint history fits in one segment in WAL work directory.
+ // Will start from this index right away.
+ curIdx = start.index();
+ }
+ }
+ else
+ curIdx = !F.isEmpty(descs) ? descs[0].idx : 0;
+
+ curIdx--;
+
+ if (log.isDebugEnabled())
+ log.debug("Initialized WAL cursor [start=" + start + ", end=" + end + ", curIdx=" + curIdx + ']');
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void advance() throws IgniteCheckedException {
+ while (true) {
+ advanceRecord();
+
+ if (curRec != null)
+ return;
+ else {
+ advanceSegment();
+
+ if (curHandle == null)
+ return;
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private void advanceRecord() {
+ try {
+ ReadFileHandle hnd = curHandle;
+
+ if (hnd != null) {
+ RecordSerializer ser = hnd.ser;
+
+ int pos = (int)hnd.in.position();
+
+ FileWALPointer ptr = new FileWALPointer(hnd.idx, pos, 0);
+
+ WALRecord rec = ser.readRecord(hnd.in, ptr);
+
+ ptr.length(rec.size());
+
+ curRec = new IgniteBiTuple<WALPointer, WALRecord>(ptr, rec);
+ }
+ }
+ catch (IOException | IgniteCheckedException e) {
+ if (!(e instanceof SegmentEofException)) {
+ if (log.isInfoEnabled())
+ log.info("Stopping WAL iteration due to an exception: " + e.getMessage());
+ }
+
+ curRec = null;
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void advanceSegment() throws IgniteCheckedException {
+ ReadFileHandle cur0 = curHandle;
+
+ if (cur0 != null) {
+ cur0.close();
+
+ if (cur0.workDir)
+ releaseWorkSegment(cur0.idx);
+
+ curHandle = null;
+ }
+
+ // We are past the end marker.
+ if (end != null && curIdx + 1 > end.index())
+ return;
+
+ curIdx++;
+
+ FileDescriptor fd;
+
+ boolean readArchive = canReadArchiveOrReserveWork(curIdx);
+
+ if (readArchive) {
+ fd = new FileDescriptor(new File(walArchiveDir,
+ FileDescriptor.fileName(curIdx)));
+ }
+ else {
+ long workIdx = curIdx % psCfg.getWalSegments();
+
+ fd = new FileDescriptor(
+ new File(walWorkDir, FileDescriptor.fileName(workIdx)),
+ curIdx);
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Reading next file [absIdx=" + curIdx + ", file=" + fd.file.getAbsolutePath() + ']');
+
+ assert fd != null;
+
+ try {
+ curHandle = initReadHandle(fd, start != null && curIdx == start.index() ? start : null);
+ }
+ catch (FileNotFoundException e) {
+ if (readArchive)
+ throw new IgniteCheckedException("Missing WAL segment in the archive", e);
+ else
+ curHandle = null;
+ }
+
+ if (curHandle != null)
+ curHandle.workDir = !readArchive;
+ else
+ releaseWorkSegment(curIdx);
+
+ curRec = null;
+ }
+
+ /**
+ * @param desc File descriptor.
+ * @param start Optional start pointer.
+ * @return Initialized file handle.
+ * @throws FileNotFoundException If segment file is missing.
+ * @throws IgniteCheckedException If initialized failed due to another unexpected error.
+ */
+ private ReadFileHandle initReadHandle(FileDescriptor desc, FileWALPointer start)
+ throws IgniteCheckedException, FileNotFoundException {
+ try {
+ RandomAccessFile rf = new RandomAccessFile(desc.file, "r");
+
+ try {
+ FileChannel channel = rf.getChannel();
+ FileInput in = new FileInput(channel, buf);
+
+ // Header record must be agnostic to the serializer version.
+ WALRecord rec = serializer.readRecord(in,
+ new FileWALPointer(desc.idx, (int)channel.position(), 0));
+
+ if (rec == null)
+ return null;
+
+ if (rec.type() != WALRecord.RecordType.HEADER_RECORD)
+ throw new IOException("Missing file header record: " + desc.file.getAbsoluteFile());
+
+ int ver = ((HeaderRecord)rec).version();
+
+ RecordSerializer ser = forVersion(cctx, ver);
+
+ if (start != null && desc.idx == start.index())
+ in.seek(start.fileOffset());
+
+ return new ReadFileHandle(rf, desc.idx, cctx.igniteInstanceName(), ser, in);
+ }
+ catch (SegmentEofException | EOFException ignore) {
+ try {
+ rf.close();
+ }
+ catch (IOException ce) {
+ throw new IgniteCheckedException(ce);
+ }
+
+ return null;
+ }
+ catch (IOException | IgniteCheckedException e) {
+ try {
+ rf.close();
+ }
+ catch (IOException ce) {
+ e.addSuppressed(ce);
+ }
+
+ throw e;
+ }
+ }
+ catch (FileNotFoundException e) {
+ throw e;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(
+ "Failed to initialize WAL segment: " + desc.file.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * @param absIdx Absolute index to check.
+ * @return {@code True} if we can safely read the archive, {@code false} if the segment has not been
+ * archived yet. In this case the corresponding work segment is reserved (will not be deleted until
+ * release).
+ */
+ private boolean canReadArchiveOrReserveWork(long absIdx) {
+ return archiver != null && archiver.checkCanReadArchiveOrReserveWorkSegment(absIdx);
+ }
+
+ /**
+ * @param absIdx Absolute index to release.
+ */
+ private void releaseWorkSegment(long absIdx) {
+ if (archiver != null)
+ archiver.releaseWorkSegment(absIdx);
+ }
+ }
+
+ /**
+ * Periodically flushes current file handle for {@link WALMode#BACKGROUND} WALMode.
+ */
+ private class QueueFlusher extends Thread {
+ /** */
+ private volatile boolean stopped;
+
+ /**
+ * @param gridName Grid name.
+ */
+ private QueueFlusher(String gridName) {
+ super("wal-queue-flusher-#" + gridName);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ while (!stopped) {
+ long wakeup = U.currentTimeMillis() + flushFreq;
+
+ LockSupport.parkUntil(wakeup);
+
+ FileWriteHandle hnd = currentHandle();
+
+ try {
+ hnd.flush(hnd.head.get());
+ }
+ catch (IgniteCheckedException e) {
+ U.warn(log, "Failed to flush WAL record queue", e);
+ }
+ }
+ }
+
+ /**
+ * Signals stop, wakes up thread and waiting until completion.
+ */
+ private void shutdown() {
+ stopped = true;
+
+ LockSupport.unpark(this);
+
+ try {
+ join();
+ }
+ catch (InterruptedException ignore) {
+ // Got interrupted while waiting for flusher to shutdown.
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java
new file mode 100644
index 0000000..75a62a9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/RecordSerializer.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+
+/**
+ * Record serializer.
+ */
+public interface RecordSerializer {
+ /**
+ * @return serializer version
+ */
+ public int version();
+
+ /**
+ * @param record Record.
+ * @return Size in bytes.
+ */
+ public int size(WALRecord record) throws IgniteCheckedException;
+
+ /**
+ * @param record Entry to write.
+ * @param buf Buffer.
+ */
+ public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException;
+
+ /**
+ * @param in Data input to read data from.
+ * @return Read entry.
+ */
+ public WALRecord readRecord(FileInput in, WALPointer expPtr) throws IOException, IgniteCheckedException;
+}
[05/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoveryTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoveryTest.java
deleted file mode 100644
index 4eb3d34..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoveryTest.java
+++ /dev/null
@@ -1,1288 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteCompute;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
-import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.tree.io.TrackingPageIO;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.PA;
-import org.apache.ignite.internal.util.typedef.PAX;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.resources.IgniteInstanceResource;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
-import org.junit.Assert;
-import sun.nio.ch.DirectBuffer;
-
-/**
- *
- */
-public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
- /** */
- private static final String HAS_CACHE = "HAS_CACHE";
-
- /** */
- private static final int LARGE_ARR_SIZE = 1025;
-
- /** */
- private boolean fork;
-
- /** */
- private String cacheName;
-
- /** */
- private int walSegmentSize;
-
- /** Logger only. */
- private boolean logOnly;
-
- /** {@inheritDoc} */
- @Override protected boolean isMultiJvm() {
- return fork;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(cacheName);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
- ccfg.setNodeFilter(new RemoteNodeFilter());
- ccfg.setIndexedTypes(Integer.class, IndexedObject.class);
-
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- dbCfg.setPageSize(4 * 1024);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(1024 * 1024 * 1024);
- memPlcCfg.setMaxSize(1024 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(dbCfg);
-
- PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
-
- if (logOnly)
- pCfg.setWalMode(WALMode.LOG_ONLY);
-
- if (walSegmentSize != 0)
- pCfg.setWalSegmentSize(walSegmentSize);
-
- cfg.setPersistentStoreConfiguration(pCfg);
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration binCfg = new BinaryConfiguration();
-
- binCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(binCfg);
-
- if (!getTestIgniteInstanceName(0).equals(gridName))
- cfg.setUserAttributes(F.asMap(HAS_CACHE, true));
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- cacheName = "partitioned";
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- logOnly = false;
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalBig() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- Random rnd = new Random();
-
- Map<Integer, IndexedObject> map = new HashMap<>();
-
- for (int i = 0; i < 10_000; i++) {
- if (i % 1000 == 0)
- X.println(" >> " + i);
-
- int k = rnd.nextInt(300_000);
- IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
-
- cache.put(k, v);
- map.put(k, v);
- }
-
- // Check.
- for (Integer k : map.keySet())
- assertEquals(map.get(k), cache.get(k));
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- cache = ignite.cache("partitioned");
-
- // Check.
- for (Integer k : map.keySet())
- assertEquals(map.get(k), cache.get(k));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testSwitchClassLoader() throws Exception {
- try {
- final IgniteEx igniteEx = startGrid(1);
-
- // CustomDiscoveryMessage will trigger service tasks
- startGrid(2);
-
- IgniteCache<Integer, EnumVal> cache = igniteEx.cache("partitioned");
-
- // Creates LoadCacheJobV2
-// cache.loadCache(null);
-
- final ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
- final ClassLoader newCl = getExternalClassLoader();
-
- Thread.currentThread().setContextClassLoader(newCl);
-
- for (int i = 0; i < 10; i++)
- cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
-
- for (int i = 0; i < 10; i++)
- assert cache.containsKey(i);
-
- // Invokes ClearTask with new class loader
- cache.clear();
-
- Thread.currentThread().setContextClassLoader(oldCl);
-
- for (int i = 0; i < 10; i++)
- cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
-
- for (int i = 0; i < 10; i++)
- assert cache.containsKey(i);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalSimple() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- info(" --> step1");
-
- for (int i = 0; i < 10_000; i += 2) {
-// X.println(" -> put: " + i);
-
- cache.put(i, new IndexedObject(i));
- }
-
- info(" --> step2");
-
- for (int i = 0; i < 10_000; i += 3)
- cache.put(i, new IndexedObject(i * 2));
-
- info(" --> step3");
-
- for (int i = 0; i < 10_000; i += 7)
- cache.put(i, new IndexedObject(i * 3));
-
- info(" --> check1");
-
- // Check.
- for (int i = 0; i < 10_000; i++) {
- IndexedObject o;
-
- if (i % 7 == 0)
- o = new IndexedObject(i * 3);
- else if (i % 3 == 0)
- o = new IndexedObject(i * 2);
- else if (i % 2 == 0)
- o = new IndexedObject(i);
- else
- o = null;
-
- assertEquals(o, cache.get(i));
- }
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- cache = ignite.cache("partitioned");
-
- info(" --> check2");
-
- // Check.
- for (int i = 0; i < 10_000; i++) {
- IndexedObject o;
-
- if (i % 7 == 0)
- o = new IndexedObject(i * 3);
- else if (i % 3 == 0)
- o = new IndexedObject(i * 2);
- else if (i % 2 == 0)
- o = new IndexedObject(i);
- else
- o = null;
-
- assertEquals(o, cache.get(i));
- }
-
- info(" --> ok");
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testWalLargeValue() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++) {
- final byte[] data = new byte[i];
-
- Arrays.fill(data, (byte)i);
-
- cache.put(i, data);
-
- if (i % 1000 == 0)
- X.println(" ---> put: " + i);
-
-// Assert.assertArrayEquals(data, (byte[])cache.get(i));
- }
-
-// info(" --> check1");
-//
-// for (int i = 0; i < 25_000; i++) {
-// final byte[] data = new byte[i];
-//
-// Arrays.fill(data, (byte)i);
-//
-// final byte[] loaded = (byte[]) cache.get(i);
-//
-// Assert.assertArrayEquals(data, loaded);
-// }
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- cache = ignite.cache("partitioned");
-
- info(" --> check2");
-
- for (int i = 0; i < 10_000; i++) {
- final byte[] data = new byte[i];
-
- Arrays.fill(data, (byte)i);
-
- final byte[] loaded = (byte[]) cache.get(i);
-
- Assert.assertArrayEquals(data, loaded);
-
- if (i % 1000 == 0)
- X.println(" ---> get: " + i);
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRolloverMultithreadedDefault() throws Exception {
- logOnly = false;
-
- checkWalRolloverMultithreaded();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRolloverMultithreadedLogOnly() throws Exception {
- logOnly = true;
-
- checkWalRolloverMultithreaded();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testHugeCheckpointRecord() throws Exception {
- try {
- final IgniteEx ignite = startGrid(1);
-
- for (int i = 0; i < 50; i++) {
- CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>("cache-" + i);
-
- // We can get 'too many open files' with default number of partitions.
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 128));
-
- IgniteCache<Object, Object> cache = ignite.getOrCreateCache(ccfg);
-
- cache.put(i, i);
- }
-
- final long endTime = System.currentTimeMillis() + 30_000;
-
- IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- Random rnd = ThreadLocalRandom.current();
-
- while (U.currentTimeMillis() < endTime) {
- IgniteCache<Object, Object> cache = ignite.cache("cache-" + rnd.nextInt(50));
-
- cache.put(rnd.nextInt(50_000), rnd.nextInt());
- }
-
- return null;
- }
- }, 16, "put-thread");
-
- while (System.currentTimeMillis() < endTime) {
- ignite.context().cache().context().database().wakeupForCheckpoint("test").get();
-
- U.sleep(500);
- }
-
- fut.get();
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- private void checkWalRolloverMultithreaded() throws Exception {
- walSegmentSize = 2 * 1024 * 1024;
-
- final long endTime = System.currentTimeMillis() + 2 * 60 * 1000;
-
- try {
- IgniteEx ignite = startGrid(1);
-
- final IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- GridTestUtils.runMultiThreaded(new Callable<Void>() {
- @Override public Void call() throws Exception {
- Random rnd = ThreadLocalRandom.current();
-
- while (U.currentTimeMillis() < endTime)
- cache.put(rnd.nextInt(50_000), rnd.nextInt());
-
- return null;
- }
- }, 16, "put-thread");
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testWalRenameDirSimple() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 100; i++)
- cache.put(i, new IndexedObject(i));
-
- stopGrid(1);
-
- final File cacheDir = cacheDir("partitioned", ignite.context().discovery().consistentId().toString());
-
- final boolean renamed = cacheDir.renameTo(new File(cacheDir.getParent(), "cache-partitioned0"));
-
- assert renamed;
-
- cacheName = "partitioned0";
-
- ignite = startGrid(1);
-
- cache = ignite.cache(cacheName);
-
- for (int i = 0; i < 100; i++)
- assertEquals(new IndexedObject(i), cache.get(i));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param cacheName Cache name.
- * @param consId Consistent ID.
- * @return Cache dir.
- * @throws IgniteCheckedException If fail.
- */
- private File cacheDir(final String cacheName, String consId) throws IgniteCheckedException {
- consId = consId.replaceAll("[\\.:]", "_");
-
- final File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
-
- assert dbDir.exists();
-
- final File consIdDir = new File(dbDir.getAbsolutePath(), consId);
-
- assert consIdDir.exists();
-
- final File cacheDir = new File(consIdDir.getAbsolutePath(), "cache-" + cacheName);
-
- assert cacheDir.exists();
-
- return cacheDir;
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRecoveryNoCheckpoint() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LoadRunnable(false));
-
- info("Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- final IgniteEx g0 = ctrlGrid;
-
- GridTestUtils.waitForCondition(new PA() {
- /** {@inheritDoc} */
- @Override public boolean apply() {
- return g0.cluster().nodes().size() == 1;
- }
- }, getTestTimeout());
-
- fork = false;
-
- // Now start the grid and verify that updates were restored from WAL.
- cacheGrid = startGrid(1);
-
- IgniteCache<Object, Object> cache = cacheGrid.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++)
- assertEquals(new IndexedObject(i), cache.get(i));
-
- List<List<?>> res = cache.query(new SqlFieldsQuery("select count(iVal) from IndexedObject")).getAll();
-
- assertEquals(1, res.size());
- assertEquals(10_000L, res.get(0).get(0));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRecoveryLargeNoCheckpoint() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LargeLoadRunnable(false));
-
- info("Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- final IgniteEx g0 = ctrlGrid;
-
- GridTestUtils.waitForCondition(new PA() {
- /** {@inheritDoc} */
- @Override public boolean apply() {
- return g0.cluster().nodes().size() == 1;
- }
- }, getTestTimeout());
-
- fork = false;
-
- // Now start the grid and verify that updates were restored from WAL.
- cacheGrid = startGrid(1);
-
- IgniteCache<Object, Object> cache = cacheGrid.cache("partitioned");
-
- for (int i = 0; i < 1000; i++) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- Arrays.fill(data, i);
-
- final long[] loaded = (long[]) cache.get(i);
-
- Assert.assertArrayEquals(data, loaded);
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return TimeUnit.MINUTES.toMillis(20);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRandomCrash() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
-
- rmt.run(new LoadRunnable(false));
-
- info(">>> Finished cache population.");
-
- rmt.run(new AsyncLoadRunnable());
-
- Thread.sleep(20_000);
-
- info(">>> Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- startGrid(1);
-
- Boolean res = rmt.call(new VerifyCallable());
-
- assertTrue(res);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testLargeRandomCrash() throws Exception {
- try {
- IgniteEx ctrlGrid = startGrid(0);
-
- fork = true;
-
- IgniteEx cacheGrid = startGrid(1);
-
- IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
-
- rmt.run(new LargeLoadRunnable(false));
-
- info(">>> Finished cache population.");
-
- rmt.run(new AsyncLargeLoadRunnable());
-
- Thread.sleep(20_000);
-
- info(">>> Killing remote process...");
-
- ((IgniteProcessProxy)cacheGrid).kill();
-
- startGrid(1);
-
- Boolean res = rmt.call(new VerifyLargeCallable());
-
- assertTrue(res);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- *
- */
- private static class RemoteNodeFilter implements IgnitePredicate<ClusterNode> {
- /** {@inheritDoc} */
- @Override public boolean apply(ClusterNode clusterNode) {
- return clusterNode.attribute(HAS_CACHE) != null;
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testDestroyCache() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- IgniteCache<Object, Object> cache = ignite.getOrCreateCache("test");
-
- cache.put(1, new IndexedObject(1));
-
- ignite.destroyCache("test");
-
- cache = ignite.getOrCreateCache("test");
-
- // No entry available after cache destroy.
- assertNull(cache.get(1));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testEvictPartition() throws Exception {
- try {
- Ignite ignite1 = startGrid("node1");
-
- IgniteCache<Object, Object> cache1 = ignite1.cache(cacheName);
-
- for (int i = 0; i < 100; i++)
- cache1.put(i, new IndexedObject(i));
-
- Ignite ignite2 = startGrid("node2");
-
- IgniteCache<Object, Object> cache2 = ignite2.cache(cacheName);
-
- for (int i = 0; i < 100; i++) {
- assertEquals(new IndexedObject(i), cache1.get(i));
- assertEquals(new IndexedObject(i), cache2.get(i));
- }
-
- ignite1.close();
- ignite2.close();
-
- ignite1 = startGrid("node1");
- ignite2 = startGrid("node2");
-
- cache1 = ignite1.cache(cacheName);
- cache2 = ignite2.cache(cacheName);
-
- for (int i = 0; i < 100; i++) {
- assertEquals(new IndexedObject(i), cache1.get(i));
- assertEquals(new IndexedObject(i), cache2.get(i));
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testApplyDeltaRecords() throws Exception {
- try {
- IgniteEx ignite0 = (IgniteEx)startGrid("node0");
-
- IgniteCache<Object, Object> cache0 = ignite0.cache(cacheName);
-
- for (int i = 0; i < 1000; i++)
- cache0.put(i, new IndexedObject(i));
-
- GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
-
- db.waitForCheckpoint("test");
- db.enableCheckpoints(false).get();
-
- // Log something to know where to start.
- WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
-
- info("Replay marker: " + ptr);
-
- for (int i = 1000; i < 5000; i++)
- cache0.put(i, new IndexedObject(i));
-
- info("Done puts...");
-
- for (int i = 2_000; i < 3_000; i++)
- cache0.remove(i);
-
- info("Done removes...");
-
- for (int i = 5000; i < 6000; i++)
- cache0.put(i, new IndexedObject(i));
-
- info("Done puts...");
-
- Map<FullPageId, byte[]> rolledPages = new HashMap<>();
-
- int pageSize = sharedCtx.database().pageSize();
-
- ByteBuffer buf1 = ByteBuffer.allocateDirect(pageSize);
-
- // Now check that deltas can be correctly applied.
- try (WALIterator it = sharedCtx.wal().replay(ptr)) {
- while (it.hasNext()) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
-
- WALRecord rec = tup.get2();
-
- if (rec instanceof PageSnapshot) {
- PageSnapshot page = (PageSnapshot)rec;
-
- rolledPages.put(page.fullPageId(), page.pageData());
- }
- else if (rec instanceof PageDeltaRecord) {
- PageDeltaRecord delta = (PageDeltaRecord)rec;
-
- FullPageId fullId = new FullPageId(delta.pageId(), delta.cacheId());
-
- byte[] pageData = rolledPages.get(fullId);
-
- if (pageData == null) {
- pageData = new byte[pageSize];
-
- rolledPages.put(fullId, pageData);
- }
-
- assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
-
- buf1.order(ByteOrder.nativeOrder());
-
- buf1.position(0);
- buf1.put(pageData);
- buf1.position(0);
-
- delta.applyDelta(sharedCtx
- .database()
- .memoryPolicy(null)
- .pageMemory(),
-
- ((DirectBuffer)buf1).address());
-
- buf1.position(0);
-
- buf1.get(pageData);
- }
- }
- }
-
- info("Done apply...");
-
- PageMemoryEx pageMem = (PageMemoryEx)db.memoryPolicy(null).pageMemory();
-
- for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
- FullPageId fullId = entry.getKey();
-
- ignite0.context().cache().context().database().checkpointReadLock();
-
- try {
- long page = pageMem.acquirePage(fullId.cacheId(), fullId.pageId(), true);
-
- try {
- long buf = pageMem.writeLock(fullId.cacheId(), fullId.pageId(), page, true);
-
- try {
- byte[] data = entry.getValue();
-
- for (int i = 0; i < data.length; i++) {
- if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
- continue; // Skip tracking pages.
-
- assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(buf, i), data[i]);
- }
- }
- finally {
- pageMem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, false, true);
- }
- }
- finally {
- pageMem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- finally {
- ignite0.context().cache().context().database().checkpointReadUnlock();
- }
- }
-
- ignite0.close();
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- *
- */
- private static class LoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** */
- private boolean disableCheckpoints;
-
- /**
- * @param disableCheckpoints Disable checkpoints flag.
- */
- private LoadRunnable(boolean disableCheckpoints) {
- this.disableCheckpoints = disableCheckpoints;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- ignite.log().info("Started load.");
-
- if (disableCheckpoints) {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- try {
- dbMgr.enableCheckpoints(false).get();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++)
- cache.put(i, new IndexedObject(i));
-
- ignite.log().info("Finished load.");
- }
- }
-
- /**
- *
- */
- private static class AsyncLoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ignite.log().info(">>>>>>> Started load.");
-
- for (int i = 0; i < 4; i++) {
- ignite.scheduler().callLocal(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- int cnt = 0;
-
- while (!Thread.currentThread().isInterrupted()) {
- cache.put(rnd.nextInt(10_000), new IndexedObject(rnd.nextInt()));
-
- cnt++;
-
- if (cnt > 0 && cnt % 1_000 == 0)
- ignite.log().info(">>>> Updated: " + cnt);
- }
-
- return null;
- }
- });
- }
- }
- }
-
- /**
- *
- */
- private static class VerifyCallable implements IgniteCallable<Boolean> {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 10_000; i++) {
- Object val = cache.get(i);
-
- if (val == null) {
- ignite.log().warning("Failed to find a value for key: " + i);
-
- return false;
- }
- }
-
- return true;
- }
- }
-
- /**
- *
- */
- private static class LargeLoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** */
- private boolean disableCheckpoints;
-
- /**
- * @param disableCheckpoints Disable checkpoints flag.
- */
- private LargeLoadRunnable(boolean disableCheckpoints) {
- this.disableCheckpoints = disableCheckpoints;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ignite.log().info("Started load.");
-
- if (disableCheckpoints) {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.enableCheckpoints(false);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 1000; i++) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- Arrays.fill(data, i);
-
- cache.put(i, data);
- }
-
- ignite.log().info("Finished load.");
- }
- }
-
- /**
- *
- */
- private static class AsyncLargeLoadRunnable implements IgniteRunnable {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- ignite.log().info(">>>>>>> Started load.");
-
- for (int i = 0; i < 1; i++) {
- ignite.scheduler().callLocal(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- int cnt = 0;
-
- while (!Thread.currentThread().isInterrupted()) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- final int key = rnd.nextInt(1000);
-
- Arrays.fill(data, key);
-
-// System.out.println("> " + key);
-
- cache.put(key, data);
-
- cnt++;
-
- if (cnt > 0 && cnt % 1_000 == 0)
- ignite.log().info(">>>> Updated: " + cnt);
- }
-
- return null;
- }
- });
- }
- }
- }
-
- /**
- *
- */
- private static class VerifyLargeCallable implements IgniteCallable<Boolean> {
- /** */
- @IgniteInstanceResource
- private Ignite ignite;
-
- /** {@inheritDoc} */
- @Override public Boolean call() throws Exception {
- try {
- boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- return ignite.cache("partitioned") != null;
- }
- }, 10_000);
-
- assertTrue(successfulWaiting);
- }
- catch (IgniteInterruptedCheckedException e) {
- throw new RuntimeException(e);
- }
-
- IgniteCache<Object, Object> cache = ignite.cache("partitioned");
-
- for (int i = 0; i < 1000; i++) {
- final long[] data = new long[LARGE_ARR_SIZE];
-
- Arrays.fill(data, i);
-
- final Object val = cache.get(i);
-
- if (val == null) {
- ignite.log().warning("Failed to find a value for key: " + i);
-
- return false;
- }
- }
-
- return true;
- }
- }
-
-
- /**
- *
- */
- private static class IndexedObject {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /**
- * @param iVal Integer value.
- */
- private IndexedObject(int iVal) {
- this.iVal = iVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof IndexedObject))
- return false;
-
- IndexedObject that = (IndexedObject)o;
-
- return iVal == that.iVal;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return iVal;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IndexedObject.class, this);
- }
- }
-
- /**
- *
- */
- private enum EnumVal {
- /** */
- VAL1,
-
- /** */
- VAL2,
-
- /** */
- VAL3
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/WalRecoveryTxLogicalRecordsTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/WalRecoveryTxLogicalRecordsTest.java
deleted file mode 100644
index 30a7f9e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/WalRecoveryTxLogicalRecordsTest.java
+++ /dev/null
@@ -1,951 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReferenceArray;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.Ignition;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.QueryCursor;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.store.PageStore;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
-import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeListImpl;
-import org.apache.ignite.internal.processors.cache.database.freelist.PagesList;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseListImpl;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.Transaction;
-import org.junit.Assert;
-
-/**
- *
- */
-public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
- /** Cache name. */
- private static final String CACHE_NAME = "cache";
-
- /** Cache 2 name. */
- private static final String CACHE2_NAME = "cache2";
-
- /** */
- public static final int PARTS = 32;
-
- /** */
- public static final int WAL_HIST_SIZE = 30;
-
- /** */
- private int pageSize = 4 * 1024;
-
- /** */
- private CacheConfiguration<?, ?> extraCcfg;
-
- /** */
- private Long checkpointFreq;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration<Integer, IndexedValue> ccfg = new CacheConfiguration<>(CACHE_NAME);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
- ccfg.setIndexedTypes(Integer.class, IndexedValue.class);
-
- if (extraCcfg != null)
- cfg.setCacheConfiguration(ccfg, new CacheConfiguration<>(extraCcfg));
- else
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- dbCfg.setPageSize(pageSize);
-
- cfg.setMemoryConfiguration(dbCfg);
-
- PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
-
- pCfg.setWalHistorySize(WAL_HIST_SIZE);
-
- if (checkpointFreq != null)
- pCfg.setCheckpointingFrequency(checkpointFreq);
-
- cfg.setPersistentStoreConfiguration(pCfg);
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration binCfg = new BinaryConfiguration();
-
- binCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(binCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testWalTxSimple() throws Exception {
- Ignite ignite = startGrid();
-
- try {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.enableCheckpoints(false).get();
-
- IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
-
- int txCnt = 100;
-
- int keysPerTx = 10;
-
- for (int i = 0; i < txCnt; i++) {
- try (Transaction tx = ignite.transactions().txStart()) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- cache.put(k, new IndexedValue(k));
- }
-
- tx.commit();
- }
- }
-
- for (int i = 0; i < txCnt; i++) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- assertEquals(k, cache.get(k).value());
- }
- }
-
- stopGrid();
-
- ignite = startGrid();
-
- cache = ignite.cache(CACHE_NAME);
-
- for (int i = 0; i < txCnt; i++) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- assertEquals(k, cache.get(k).value());
- }
- }
-
- for (int i = 0; i < txCnt; i++) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- QueryCursor<List<?>> cur = cache.query(
- new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
-
- List<List<?>> vals = cur.getAll();
-
- assertEquals(vals.size(), 1);
- assertEquals("string-" + k, vals.get(0).get(0));
- }
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRecoveryRemoves() throws Exception {
- Ignite ignite = startGrid();
-
- try {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
-
- int txCnt = 100;
-
- int keysPerTx = 10;
-
- for (int i = 0; i < txCnt; i++) {
- try (Transaction tx = ignite.transactions().txStart()) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- cache.put(k, new IndexedValue(k));
- }
-
- tx.commit();
- }
- }
-
- for (int i = 0; i < txCnt; i++) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- assertEquals(k, cache.get(k).value());
- }
- }
-
- dbMgr.waitForCheckpoint("test");
- dbMgr.enableCheckpoints(false).get();
-
- for (int i = 0; i < txCnt / 2; i++) {
- try (Transaction tx = ignite.transactions().txStart()) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- cache.remove(k);
- }
-
- tx.commit();
- }
- }
-
- stopGrid();
-
- ignite = startGrid();
-
- cache = ignite.cache(CACHE_NAME);
-
- for (int i = 0; i < txCnt; i++) {
- for (int j = 0; j < keysPerTx; j++) {
- int k = i * keysPerTx + j;
-
- QueryCursor<List<?>> cur = cache.query(
- new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
-
- List<List<?>> vals = cur.getAll();
-
- if (i < txCnt / 2) {
- assertNull(cache.get(k));
- assertTrue(F.isEmpty(vals));
- }
- else {
- assertEquals(k, cache.get(k).value());
-
- assertEquals(1, vals.size());
- assertEquals("string-" + k, vals.get(0).get(0));
- }
- }
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalanceIterator() throws Exception {
- extraCcfg = new CacheConfiguration(CACHE2_NAME);
- extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
-
- Ignite ignite = startGrid();
-
- try {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.waitForCheckpoint("test");
-
- // This number depends on wal history size.
- int entries = 25;
-
- IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
- IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE2_NAME);
-
- for (int i = 0; i < entries; i++) {
- // Put to partition 0.
- cache.put(i * PARTS, i * PARTS);
-
- // Put to partition 1.
- cache.put(i * PARTS + 1, i * PARTS + 1);
-
- // Put to another cache.
- cache2.put(i, i);
-
- dbMgr.waitForCheckpoint("test");
- }
-
- for (int i = 0; i < entries; i++) {
- assertEquals((Integer)(i * PARTS), cache.get(i * PARTS));
- assertEquals((Integer)(i * PARTS + 1), cache.get(i * PARTS + 1));
- assertEquals((Integer)(i), cache2.get(i));
- }
-
- GridCacheContext<Object, Object> cctx = ((IgniteEx)ignite).context().cache().cache(CACHE_NAME).context();
- IgniteCacheOffheapManager offh = cctx.offheap();
- AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion();
-
- for (int i = 0; i < entries; i++) {
- try (IgniteRebalanceIterator it = offh.rebalanceIterator(0, topVer, (long)i)) {
- assertTrue("Not historical for iteration: " + i, it.historical());
-
- assertNotNull(it);
-
- for (int j = i; j < entries; j++) {
- assertTrue("i=" + i + ", j=" + j, it.hasNextX());
-
- CacheDataRow row = it.next();
-
- assertEquals(j * PARTS, (int)row.key().value(cctx.cacheObjectContext(), false));
- assertEquals(j * PARTS, (int)row.value().value(cctx.cacheObjectContext(), false));
- }
-
- assertFalse(it.hasNext());
- }
-
- try (IgniteRebalanceIterator it = offh.rebalanceIterator(1, topVer, (long)i)) {
- assertNotNull(it);
-
- assertTrue("Not historical for iteration: " + i, it.historical());
-
- for (int j = i; j < entries; j++) {
- assertTrue(it.hasNextX());
-
- CacheDataRow row = it.next();
-
- assertEquals(j * PARTS + 1, (int)row.key().value(cctx.cacheObjectContext(), false));
- assertEquals(j * PARTS + 1, (int)row.value().value(cctx.cacheObjectContext(), false));
- }
-
- assertFalse(it.hasNext());
- }
- }
-
- stopAllGrids();
-
- // Check that iterator is valid after restart.
- ignite = startGrid();
-
- cctx = ((IgniteEx)ignite).context().cache().cache(CACHE_NAME).context();
- offh = cctx.offheap();
- topVer = cctx.affinity().affinityTopologyVersion();
-
- for (int i = 0; i < entries; i++) {
- long start = System.currentTimeMillis();
-
- try (IgniteRebalanceIterator it = offh.rebalanceIterator(0, topVer, (long)i)) {
- long end = System.currentTimeMillis();
-
- info("Time to get iterator: " + (end - start));
-
- assertTrue("Not historical for iteration: " + i, it.historical());
-
- assertNotNull(it);
-
- start = System.currentTimeMillis();
-
- for (int j = i; j < entries; j++) {
- assertTrue("i=" + i + ", j=" + j, it.hasNextX());
-
- CacheDataRow row = it.next();
-
- assertEquals(j * PARTS, (int)row.key().value(cctx.cacheObjectContext(), false));
- assertEquals(j * PARTS, (int)row.value().value(cctx.cacheObjectContext(), false));
- }
-
- end = System.currentTimeMillis();
-
- info("Time to iterate: " + (end - start));
-
- assertFalse(it.hasNext());
- }
-
- try (IgniteRebalanceIterator it = offh.rebalanceIterator(1, topVer, (long)i)) {
- assertNotNull(it);
-
- assertTrue("Not historical for iteration: " + i, it.historical());
-
- for (int j = i; j < entries; j++) {
- assertTrue(it.hasNextX());
-
- CacheDataRow row = it.next();
-
- assertEquals(j * PARTS + 1, (int)row.key().value(cctx.cacheObjectContext(), false));
- assertEquals(j * PARTS + 1, (int)row.value().value(cctx.cacheObjectContext(), false));
- }
-
- assertFalse(it.hasNext());
- }
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testCheckpointHistory() throws Exception {
- Ignite ignite = startGrid();
-
- try {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.waitForCheckpoint("test");
-
- // This number depends on wal history size.
- int entries = WAL_HIST_SIZE * 2;
-
- IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
-
- for (int i = 0; i < entries; i++) {
- // Put to partition 0.
- cache.put(i * PARTS, i * PARTS);
-
- // Put to partition 1.
- cache.put(i * PARTS + 1, i * PARTS + 1);
-
- dbMgr.waitForCheckpoint("test");
- }
-
- GridCacheDatabaseSharedManager.CheckpointHistory hist = dbMgr.checkpointHistory();
-
- assertTrue(hist.checkpoints().size() <= WAL_HIST_SIZE);
-
- File cpDir = dbMgr.checkpointDirectory();
-
- File[] cpFiles = cpDir.listFiles();
-
- assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testWalAfterPreloading() throws Exception {
- Ignite ignite = startGrid();
-
- try {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.enableCheckpoints(false).get();
-
- int entries = 100;
-
- try (IgniteDataStreamer<Integer, Integer> streamer = ignite.dataStreamer(CACHE_NAME)) {
- for (int i = 0; i < entries; i++)
- streamer.addData(i, i);
- }
-
- IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
-
- for (int i = 0; i < entries; i++)
- assertEquals(new Integer(i), cache.get(i));
-
- stopGrid();
-
- ignite = startGrid();
-
- cache = ignite.cache(CACHE_NAME);
-
- for (int i = 0; i < entries; i++)
- assertEquals(new Integer(i), cache.get(i));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testRecoveryRandomPutRemove() throws Exception {
- try {
- pageSize = 1024;
-
- extraCcfg = new CacheConfiguration(CACHE2_NAME);
- extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
-
- Ignite ignite = startGrid(0);
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.enableCheckpoints(false).get();
-
- IgniteCache<Integer, IndexedValue> cache1 = ignite.cache(CACHE_NAME);
- IgniteCache<Object, Object> cache2 = ignite.cache(CACHE2_NAME);
-
- final int KEYS1 = 100;
-
- for (int i = 0; i < KEYS1; i++)
- cache1.put(i, new IndexedValue(i));
-
- for (int i = 0; i < KEYS1; i++) {
- if (i % 2 == 0)
- cache1.remove(i);
- }
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < KEYS1; i++) {
- cache2.put(i, new byte[rnd.nextInt(512)]);
-
- if (rnd.nextBoolean())
- cache2.put(i, new byte[rnd.nextInt(512)]);
-
- if (rnd.nextBoolean())
- cache2.remove(i);
- }
-
- ignite.close();
-
- ignite = startGrid(0);
-
- ignite.cache(CACHE_NAME).put(1, new IndexedValue(0));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testRecoveryNoPageLost1() throws Exception {
- recoveryNoPageLost(false);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testRecoveryNoPageLost2() throws Exception {
- recoveryNoPageLost(true);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testRecoveryNoPageLost3() throws Exception {
- try {
- pageSize = 1024;
- checkpointFreq = 100L;
- extraCcfg = new CacheConfiguration(CACHE2_NAME);
- extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- List<Integer> pages = null;
-
- for (int iter = 0; iter < 5; iter++) {
- log.info("Start node: " + iter);
-
- Ignite ignite = startGrid(0);
-
- if (pages != null) {
- List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
-
- assertEquals("Iter = " + iter, pages, curPags);
- }
-
- final IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
-
- final int ops = ThreadLocalRandom.current().nextInt(10) + 10;
-
- GridTestUtils.runMultiThreaded(new Callable<Void>() {
- @Override public Void call() throws Exception {
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < ops; i++) {
- Integer key = rnd.nextInt(1000);
-
- cache.put(key, new byte[rnd.nextInt(512)]);
-
- if (rnd.nextBoolean())
- cache.remove(key);
- }
-
- return null;
- }
- }, 10, "update");
-
- pages = allocatedPages(ignite, CACHE2_NAME);
-
- Ignition.stop(ignite.name(), false); //will make checkpoint
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param checkpoint Checkpoint enable flag.
- * @throws Exception If failed.
- */
- private void recoveryNoPageLost(boolean checkpoint) throws Exception {
- try {
- pageSize = 1024;
- extraCcfg = new CacheConfiguration(CACHE2_NAME);
- extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- List<Integer> pages = null;
-
- AtomicInteger cnt = new AtomicInteger();
-
- for (int iter = 0; iter < 5; iter++) {
- log.info("Start node: " + iter);
-
- Ignite ignite = startGrid(0);
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- if (!checkpoint)
- dbMgr.enableCheckpoints(false).get();
-
- if (pages != null) {
- List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
-
- assertEquals(pages, curPags);
- }
-
- IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
-
- for (int i = 0; i < 128; i++)
- cache.put(cnt.incrementAndGet(), new byte[256 + iter * 100]);
-
- pages = allocatedPages(ignite, CACHE2_NAME);
-
- ignite.close();
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param ignite Node.
- * @param cacheName Cache name.
- * @return Allocated pages per-store.
- * @throws Exception If failed.
- */
- private List<Integer> allocatedPages(Ignite ignite, String cacheName) throws Exception {
- FilePageStoreManager storeMgr =
- (FilePageStoreManager)((IgniteEx)ignite).context().cache().context().pageStore();
-
- int parts = ignite.affinity(cacheName).partitions();
-
- List<Integer> res = new ArrayList<>(parts);
-
- for (int p = 0; p < parts; p++) {
- PageStore store = storeMgr.getStore(CU.cacheId(cacheName), p);
-
- store.sync();
-
- res.add(store.pages());
- }
-
- PageStore store = storeMgr.getStore(CU.cacheId(cacheName), PageIdAllocator.INDEX_PARTITION);
-
- store.sync();
-
- res.add(store.pages());
-
- return res;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testFreeListRecovery() throws Exception {
- try {
- pageSize = 1024;
- extraCcfg = new CacheConfiguration(CACHE2_NAME);
-
- Ignite ignite = startGrid(0);
-
- IgniteCache<Integer, IndexedValue> cache1 = ignite.cache(CACHE_NAME);
- IgniteCache<Object, Object> cache2 = ignite.cache(CACHE2_NAME);
-
- final int KEYS1 = 2048;
-
- for (int i = 0; i < KEYS1; i++)
- cache1.put(i, new IndexedValue(i));
-
- for (int i = 0; i < KEYS1; i++) {
- if (i % 2 == 0)
- cache1.remove(i);
- }
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- for (int i = 0; i < KEYS1; i++) {
- cache2.put(i, new byte[rnd.nextInt(512)]);
-
- if (rnd.nextBoolean())
- cache2.put(i, new byte[rnd.nextInt(512)]);
-
- if (rnd.nextBoolean())
- cache2.remove(i);
- }
-
- Map<Integer, T2<Map<Integer, long[]>, int[]>> cache1_1 = getFreeListData(ignite, CACHE_NAME);
- Map<Integer, T2<Map<Integer, long[]>, int[]>> cache2_1 = getFreeListData(ignite, CACHE2_NAME);
- T2<long[], Integer> rl1_1 = getReuseListData(ignite, CACHE_NAME);
- T2<long[], Integer> rl2_1 = getReuseListData(ignite, CACHE2_NAME);
-
- ignite.close();
-
- ignite = startGrid(0);
-
- cache1 = ignite.cache(CACHE_NAME);
- cache2 = ignite.cache(CACHE2_NAME);
-
- for (int i = 0; i < KEYS1; i++) {
- cache1.get(i);
- cache2.get(i);
- }
-
- Map<Integer, T2<Map<Integer, long[]>, int[]>> cache1_2 = getFreeListData(ignite, CACHE_NAME);
- Map<Integer, T2<Map<Integer, long[]>, int[]>> cache2_2 = getFreeListData(ignite, CACHE2_NAME);
- T2<long[], Integer> rl1_2 = getReuseListData(ignite, CACHE_NAME);
- T2<long[], Integer> rl2_2 = getReuseListData(ignite, CACHE2_NAME);
-
- checkEquals(cache1_1, cache1_2);
- checkEquals(cache2_1, cache2_2);
- checkEquals(rl1_1, rl1_2);
- checkEquals(rl2_1, rl2_2);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param ignite Node.
- * @param cacheName Cache name.
- * @return Cache reuse list data.
- */
- private T2<long[], Integer> getReuseListData(Ignite ignite, String cacheName) {
- GridCacheContext ctx = ((IgniteEx)ignite).context().cache().cache(cacheName).context();
-
- ReuseListImpl reuseList = GridTestUtils.getFieldValue(ctx.offheap(), "reuseList");
- PagesList.Stripe[] bucket = GridTestUtils.getFieldValue(reuseList, "bucket");
-
- long[] ids = null;
-
- if (bucket != null) {
- ids = new long[bucket.length];
-
- for (int i = 0; i < bucket.length; i++)
- ids[i] = bucket[i].tailId;
- }
-
-// AtomicIntegerArray cnts = GridTestUtils.getFieldValue(reuseList, PagesList.class, "cnts");
-// assertEquals(1, cnts.length());
-
- return new T2<>(ids, 0);
- }
-
- /**
- * @param rl1 Data 1 (before stop).
- * @param rl2 Data 2 (after restore).
- */
- private void checkEquals(T2<long[], Integer> rl1, T2<long[], Integer> rl2) {
- Assert.assertArrayEquals(rl1.get1(), rl2.get1());
- assertEquals(rl1.get2(), rl2.get2());
- }
-
- /**
- * @param partsLists1 Data 1 (before stop).
- * @param partsLists2 Data 2 (after restore).
- */
- private void checkEquals(Map<Integer, T2<Map<Integer, long[]>, int[]>> partsLists1,
- Map<Integer, T2<Map<Integer, long[]>, int[]>> partsLists2) {
- assertEquals(partsLists1.size(), partsLists2.size());
-
- for (Integer part : partsLists1.keySet()) {
- T2<Map<Integer, long[]>, int[]> t1 = partsLists1.get(part);
- T2<Map<Integer, long[]>, int[]> t2 = partsLists2.get(part);
-
- Map<Integer, long[]> m1 = t1.get1();
- Map<Integer, long[]> m2 = t2.get1();
-
- assertEquals(m1.size(), m2.size());
-
- for (Integer bucket : m1.keySet()) {
- long tails1[] = m1.get(bucket);
- long tails2[] = m2.get(bucket);
-
- Assert.assertArrayEquals(tails1, tails2);
- }
-
- Assert.assertArrayEquals("Wrong counts [part=" + part + ']', t1.get2(), t2.get2());
- }
- }
-
- /**
- * @param ignite Node.
- * @param cacheName Cache name.
- * @return Cache free lists data.
- */
- private Map<Integer, T2<Map<Integer, long[]>, int[]>> getFreeListData(Ignite ignite, String cacheName) {
- GridCacheContext ctx = ((IgniteEx)ignite).context().cache().cache(cacheName).context();
-
- List<GridDhtLocalPartition> parts = ctx.topology().localPartitions();
-
- assertTrue(!parts.isEmpty());
- assertEquals(ctx.affinity().partitions(), parts.size());
-
- Map<Integer, T2<Map<Integer, long[]>, int[]>> res = new HashMap<>();
-
- boolean foundNonEmpty = false;
- boolean foundTails = false;
-
- for (GridDhtLocalPartition part : parts) {
- FreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
-
- if (freeList == null)
- // Lazy store.
- continue;
-
- AtomicReferenceArray<PagesList.Stripe[]> buckets = GridTestUtils.getFieldValue(freeList,
- FreeListImpl.class, "buckets");
- //AtomicIntegerArray cnts = GridTestUtils.getFieldValue(freeList, PagesList.class, "cnts");
-
- assertNotNull(buckets);
- //assertNotNull(cnts);
- assertTrue(buckets.length() > 0);
- //assertEquals(cnts.length(), buckets.length());
-
- Map<Integer, long[]> tailsPerBucket = new HashMap<>();
-
- for (int i = 0; i < buckets.length(); i++) {
- PagesList.Stripe[] tails = buckets.get(i);
-
- long ids[] = null;
-
- if (tails != null) {
- ids = new long[tails.length];
-
- for (int j = 0; j < tails.length; j++)
- ids[j] = tails[j].tailId;
- }
-
- tailsPerBucket.put(i, ids);
-
- if (tails != null) {
- assertTrue(tails.length > 0);
-
- foundTails = true;
- }
- }
-
-// int[] cntsPerBucket = new int[cnts.length()];
-//
-// for (int i = 0; i < cnts.length(); i++) {
-// cntsPerBucket[i] = cnts.get(i);
-//
-// if (cntsPerBucket[i] > 0)
-// foundNonEmpty = true;
-// }
-
- res.put(part.id(), new T2<>(tailsPerBucket, (int[])null));
- }
-
- //assertTrue(foundNonEmpty);
- assertTrue(foundTails);
-
- return res;
- }
-
- /**
- *
- */
- private static class IndexedValue {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField
- private String sVal;
-
- /**
- * @param iVal Indexed value.
- */
- private IndexedValue(int iVal) {
- this.iVal = iVal;
- sVal = "string-" + iVal;
- }
-
- /**
- * @return Value.
- */
- private int value() {
- return iVal;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/crc/IgniteDataIntegrityTests.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/crc/IgniteDataIntegrityTests.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/crc/IgniteDataIntegrityTests.java
deleted file mode 100644
index c40a4ed..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/crc/IgniteDataIntegrityTests.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal.crc;
-
-import junit.framework.TestCase;
-import java.io.EOFException;
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.internal.processors.cache.database.wal.FileInput;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.PureJavaCrc32;
-
-/**
- *
- */
-public class IgniteDataIntegrityTests extends TestCase {
- /** File input. */
- private FileInput fileInput;
-
- /** Random access file. */
- private RandomAccessFile randomAccessFile;
-
- /** {@inheritDoc} */
- @Override protected void setUp() throws Exception {
- super.setUp();
-
- File file = File.createTempFile("integrity", "dat");
- file.deleteOnExit();
-
- randomAccessFile = new RandomAccessFile(file, "rw");
-
- fileInput = new FileInput(randomAccessFile.getChannel(), ByteBuffer.allocate(1024));
-
- PureJavaCrc32 pureJavaCrc32 = new PureJavaCrc32();
-
- ByteBuffer buf = ByteBuffer.allocate(1024);
- ThreadLocalRandom curr = ThreadLocalRandom.current();
-
- for (int i = 0; i < 1024; i+=16) {
- buf.putInt(curr.nextInt());
- buf.putInt(curr.nextInt());
- buf.putInt(curr.nextInt());
- buf.position(i);
- buf.putInt(PureJavaCrc32.calcCrc32(buf, 12));
- }
-
- randomAccessFile.write(buf.array());
- randomAccessFile.getFD().sync();
- }
-
- /**
- *
- */
- public void testSuccessfulPath() throws Exception {
- checkIntegrity();
- }
-
- /**
- *
- */
- public void testIntegrityViolationChecking() throws Exception {
- toggleOneRandomBit(0, 1024 - 16);
-
- try {
- checkIntegrity();
-
- fail();
- } catch (IgniteDataIntegrityViolationException ex) {
- //success
- }
- }
-
- /**
- *
- */
- public void testSkipingLastCorruptedEntry() throws Exception {
- toggleOneRandomBit(1024 - 16, 1024);
-
- try {
- checkIntegrity();
-
- fail();
- } catch (EOFException ex) {
- //success
- }
- }
-
- /**
- * @param rangeFrom Range from.
- * @param rangeTo Range to.
- */
- private void toggleOneRandomBit(int rangeFrom, int rangeTo) throws IOException {
- int pos = ThreadLocalRandom.current().nextInt(rangeFrom, rangeTo);
- randomAccessFile.seek(pos);
-
- byte b = randomAccessFile.readByte();
-
- b ^= (1 << 3);
-
- randomAccessFile.seek(pos);
- randomAccessFile.writeByte(b);
- randomAccessFile.getFD().sync();
- }
-
- /**
- *
- */
- private void checkIntegrity() throws Exception {
- randomAccessFile.seek(0);
-
- for (int i = 0; i < 1024 / 16; i++) {
- try(FileInput.Crc32CheckingFileInput in = fileInput.startRead(false)) {
- in.readInt();
- in.readInt();
- in.readInt();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreePageMemoryImplTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreePageMemoryImplTest.java
deleted file mode 100644
index a35903c..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreePageMemoryImplTest.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.database.BPlusTreeSelfTest;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-
-/**
- *
- */
-public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
- /** {@inheritDoc} */
- @Override protected PageMemory createPageMemory() throws Exception {
- long[] sizes = new long[CPUS + 1];
-
- for (int i = 0; i < sizes.length; i++)
- sizes[i] = 1024 * MB / CPUS;
-
- sizes[CPUS] = 10 * MB;
-
- DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
-
- GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
- new GridTestKernalContext(log),
- null,
- null,
- null,
- new NoOpPageStoreManager(),
- new NoOpWALManager(),
- new IgniteCacheDatabaseSharedManager(),
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null
- );
-
- PageMemory mem = new PageMemoryImpl(
- provider, sizes,
- sharedCtx,
- PAGE_SIZE,
- new CIX3<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
- assert false : "No evictions should happen during the test";
- }
- },
- new CIX3<Long, FullPageId, PageMemoryEx>(){
- @Override public void applyx(Long aLong, FullPageId fullPageId, PageMemoryEx ex) {
- }
- },
- new CheckpointLockStateChecker() {
- @Override public boolean checkpointLockIsHeldByThread() {
- return true;
- }
- },
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
-
- mem.start();
-
- return mem;
- }
-
- /** {@inheritDoc} */
- @Override protected long acquiredPages() {
- return ((PageMemoryImpl)pageMem).acquiredPages();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
deleted file mode 100644
index 5c4502c..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-
-/**
- *
- */
-public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest {
- /** {@inheritDoc} */
- @Override protected PageMemory createPageMemory() throws Exception {
- long[] sizes = new long[CPUS + 1];
-
- for (int i = 0; i < sizes.length; i++)
- sizes[i] = 1024 * MB / CPUS;
-
- sizes[CPUS] = 10 * MB;
-
- DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
-
- GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
- new GridTestKernalContext(log),
- null,
- null,
- null,
- new NoOpPageStoreManager(),
- new NoOpWALManager(),
- new IgniteCacheDatabaseSharedManager(),
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null
- );
-
- PageMemory mem = new PageMemoryImpl(
- provider, sizes,
- sharedCtx,
- PAGE_SIZE,
- new CIX3<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
- assert false : "No evictions should happen during the test";
- }
- },
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(Long page, FullPageId fullPageId, PageMemoryEx pageMem) {
- }
- }, new CheckpointLockStateChecker() {
- @Override public boolean checkpointLockIsHeldByThread() {
- return true;
- }
- },
- new MemoryMetricsImpl(new MemoryPolicyConfiguration())
- );
-
- mem.start();
-
- return mem;
- }
-
- /** {@inheritDoc} */
- @Override protected long acquiredPages() {
- return ((PageMemoryImpl)pageMem).acquiredPages();
- }
-}
[13/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryImpl.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryImpl.java
deleted file mode 100755
index 958e0ea..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryImpl.java
+++ /dev/null
@@ -1,2313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.pagemem;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.Executor;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.DirectMemoryRegion;
-import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
-import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.TrackingPageIO;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
-import org.apache.ignite.internal.util.GridLongList;
-import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.OffheapReadWriteLock;
-import org.apache.ignite.internal.util.future.CountDownFuture;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
-import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import sun.misc.JavaNioAccess;
-import sun.misc.SharedSecrets;
-import sun.nio.ch.DirectBuffer;
-
-import static java.lang.Boolean.FALSE;
-import static java.lang.Boolean.TRUE;
-
-/**
- * Page header structure is described by the following diagram.
- *
- * When page is not allocated (in a free list):
- * <pre>
- * +--------+------------------------------------------------------+
- * |8 bytes | PAGE_SIZE + PAGE_OVERHEAD - 8 bytes |
- * +--------+------------------------------------------------------+
- * |Next ptr| Page data |
- * +--------+------------------------------------------------------+
- * </pre>
- * <p/>
- * When page is allocated and is in use:
- * <pre>
- * +------------------+--------+--------+----+----+--------+--------+----------------------+
- * | 8 bytes |8 bytes |8 bytes |4 b |4 b |8 bytes |8 bytes | PAGE_SIZE |
- * +------------------+--------+--------+----+----+--------+--------+----------------------+
- * | Marker/Timestamp |Rel ptr |Page ID |C ID|PIN | LOCK |TMP BUF | Page data |
- * +------------------+--------+--------+----+----+--------+--------+----------------------+
- * </pre>
- *
- * Note that first 8 bytes of page header are used either for page marker or for next relative pointer depending
- * on whether the page is in use or not.
- */
-@SuppressWarnings({"LockAcquiredButNotSafelyReleased", "FieldAccessedSynchronizedAndUnsynchronized"})
-public class PageMemoryImpl implements PageMemoryEx {
- /** */
- public static final long PAGE_MARKER = 0x0000000000000001L;
-
- /** Relative pointer chunk index mask. */
- private static final long SEGMENT_INDEX_MASK = 0xFFFFFF0000000000L;
-
- /** Full relative pointer mask. */
- private static final long RELATIVE_PTR_MASK = 0xFFFFFFFFFFFFFFL;
-
- /** Dirty flag. */
- private static final long DIRTY_FLAG = 0x0100000000000000L;
-
- /** Dirty flag. */
- private static final long TMP_DIRTY_FLAG = 0x0200000000000000L;
-
- /** Invalid relative pointer value. */
- private static final long INVALID_REL_PTR = RELATIVE_PTR_MASK;
-
- /** */
- private static final long OUTDATED_REL_PTR = INVALID_REL_PTR + 1;
-
- /** Address mask to avoid ABA problem. */
- private static final long ADDRESS_MASK = 0xFFFFFFFFFFFFFFL;
-
- /** Counter mask to avoid ABA problem. */
- private static final long COUNTER_MASK = ~ADDRESS_MASK;
-
- /** Counter increment to avoid ABA problem. */
- private static final long COUNTER_INC = ADDRESS_MASK + 1;
-
- /** Page relative pointer. Does not change once a page is allocated. */
- public static final int RELATIVE_PTR_OFFSET = 8;
-
- /** Page ID offset */
- public static final int PAGE_ID_OFFSET = 16;
-
- /** Page cache ID offset. */
- public static final int PAGE_CACHE_ID_OFFSET = 24;
-
- /** Page pin counter offset. */
- public static final int PAGE_PIN_CNT_OFFSET = 28;
-
- /** Page lock offset. */
- public static final int PAGE_LOCK_OFFSET = 32;
-
- /** Page lock offset. */
- public static final int PAGE_TMP_BUF_OFFSET = 40;
-
- /**
- * 8b Marker/timestamp
- * 8b Relative pointer
- * 8b Page ID
- * 4b Cache ID
- * 4b Pin count
- * 8b Lock
- * 8b Temporary buffer
- */
- public static final int PAGE_OVERHEAD = 48;
-
- /** Number of random pages that will be picked for eviction. */
- public static final int RANDOM_PAGES_EVICT_NUM = 5;
-
- /** Tracking io. */
- private static final TrackingPageIO trackingIO = TrackingPageIO.VERSIONS.latest();
-
- /** Page size. */
- private final int sysPageSize;
-
- /** Shared context. */
- private final GridCacheSharedContext<?, ?> sharedCtx;
-
- /** State checker. */
- private final CheckpointLockStateChecker stateChecker;
-
- /** */
- private Executor asyncRunner = new ThreadPoolExecutor(
- 0,
- Runtime.getRuntime().availableProcessors(),
- 30L,
- TimeUnit.SECONDS,
- new ArrayBlockingQueue<Runnable>(Runtime.getRuntime().availableProcessors()));
-
- /** Page store manager. */
- private IgnitePageStoreManager storeMgr;
-
- /** */
- private IgniteWriteAheadLogManager walMgr;
-
- /** Direct byte buffer factory. */
- private JavaNioAccess nioAccess;
-
- /** */
- private final IgniteLogger log;
-
- /** Direct memory allocator. */
- private final DirectMemoryProvider directMemoryProvider;
-
- /** Segments array. */
- private Segment[] segments;
-
- /** */
- private PagePool checkpointPool;
-
- /** */
- private OffheapReadWriteLock rwLock;
-
- /** Flush dirty page closure. When possible, will be called by evictPage(). */
- private final GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage;
-
- /** Flush dirty page closure. When possible, will be called by evictPage(). */
- private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
-
- /** */
- private boolean pageEvictWarned;
-
- /** */
- private long[] sizes;
-
- /** */
- private MemoryMetricsImpl memMetrics;
-
- /**
- * @param directMemoryProvider Memory allocator to use.
- * @param sharedCtx Cache shared context.
- * @param pageSize Page size.
- * @param flushDirtyPage Callback invoked when a dirty page is evicted.
- * @param changeTracker Callback invoked to track changes in pages.
- */
- public PageMemoryImpl(
- DirectMemoryProvider directMemoryProvider,
- long[] sizes,
- GridCacheSharedContext<?, ?> sharedCtx,
- int pageSize,
- GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage,
- GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
- CheckpointLockStateChecker stateChecker,
- MemoryMetricsImpl memMetrics
- ) {
- assert sharedCtx != null;
-
- log = sharedCtx.logger(PageMemoryImpl.class);
-
- this.sharedCtx = sharedCtx;
- this.directMemoryProvider = directMemoryProvider;
- this.sizes = sizes;
- this.flushDirtyPage = flushDirtyPage;
- this.changeTracker = changeTracker;
- this.stateChecker = stateChecker;
-
- storeMgr = sharedCtx.pageStore();
- walMgr = sharedCtx.wal();
-
- assert storeMgr != null;
- assert walMgr != null;
-
- sysPageSize = pageSize + PAGE_OVERHEAD;
-
- rwLock = new OffheapReadWriteLock(128);
-
- this.memMetrics = memMetrics;
- }
-
- /** {@inheritDoc} */
- @Override public void start() throws IgniteException {
- directMemoryProvider.initialize(sizes);
-
- List<DirectMemoryRegion> regions = new ArrayList<>(sizes.length);
-
- while (true) {
- DirectMemoryRegion reg = directMemoryProvider.nextRegion();
-
- if (reg == null)
- break;
-
- regions.add(reg);
- }
-
- nioAccess = SharedSecrets.getJavaNioAccess();
-
- int regs = regions.size();
-
- segments = new Segment[regs - 1];
-
- DirectMemoryRegion cpReg = regions.get(regs - 1);
-
- checkpointPool = new PagePool(regs - 1, cpReg);
-
- long checkpointBuf = cpReg.size();
-
- long totalAllocated = 0;
- int pages = 0;
- long totalTblSize = 0;
-
- for (int i = 0; i < regs - 1; i++) {
- assert i < segments.length;
-
- DirectMemoryRegion reg = regions.get(i);
-
- totalAllocated += reg.size();
-
- segments[i] = new Segment(i, regions.get(i), checkpointPool.pages() / segments.length);
-
- pages += segments[i].pages();
- totalTblSize += segments[i].tableSize();
- }
-
- if (log.isInfoEnabled())
- log.info("Started page memory [memoryAllocated=" + U.readableSize(totalAllocated, false) +
- ", pages=" + pages +
- ", tableSize=" + U.readableSize(totalTblSize, false) +
- ", checkpointBuffer=" + U.readableSize(checkpointBuf, false) +
- ']');
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("OverlyStrongTypeCast")
- @Override public void stop() throws IgniteException {
- if (log.isDebugEnabled())
- log.debug("Stopping page memory.");
-
- directMemoryProvider.shutdown();
- }
-
- /** {@inheritDoc} */
- @Override public void releasePage(int cacheId, long pageId, long page) {
- Segment seg = segment(cacheId, pageId);
-
- seg.readLock().lock();
-
- try {
- seg.releasePage(page);
- }
- finally {
- seg.readLock().unlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public long readLock(int cacheId, long pageId, long page) {
- return readLockPage(page, new FullPageId(pageId, cacheId), false);
- }
-
- /** {@inheritDoc} */
- @Override public void readUnlock(int cacheId, long pageId, long page) {
- readUnlockPage(page);
- }
-
- /** {@inheritDoc} */
- @Override public long writeLock(int cacheId, long pageId, long page) {
- return writeLock(cacheId, pageId, page, false);
- }
-
- /** {@inheritDoc} */
- @Override public long writeLock(int cacheId, long pageId, long page, boolean restore) {
- return writeLockPage(page, new FullPageId(pageId, cacheId), !restore);
- }
-
- /** {@inheritDoc} */
- @Override public long tryWriteLock(int cacheId, long pageId, long page) {
- return tryWriteLockPage(page, new FullPageId(pageId, cacheId), true);
- }
-
- /** {@inheritDoc} */
- @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
- boolean dirtyFlag) {
- writeUnlock(cacheId, pageId, page, walPlc, dirtyFlag, false);
- }
-
- /** {@inheritDoc} */
- @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
- boolean dirtyFlag, boolean restore) {
- writeUnlockPage(page, new FullPageId(pageId, cacheId), walPlc, dirtyFlag, restore);
- }
-
- /** {@inheritDoc} */
- @Override public boolean isDirty(int cacheId, long pageId, long page) {
- return isDirty(page);
- }
-
- /** {@inheritDoc} */
- @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
- assert flags == PageIdAllocator.FLAG_DATA && partId <= PageIdAllocator.MAX_PARTITION_ID ||
- flags == PageIdAllocator.FLAG_IDX && partId == PageIdAllocator.INDEX_PARTITION :
- "flags = " + flags + ", partId = " + partId;
-
- long pageId = storeMgr.allocatePage(cacheId, partId, flags);
-
- assert PageIdUtils.pageIndex(pageId) > 0; //it's crucial for tracking pages (zero page is super one)
-
- // We need to allocate page in memory for marking it dirty to save it in the next checkpoint.
- // Otherwise it is possible that on file will be empty page which will be saved at snapshot and read with error
- // because there is no crc inside them.
- Segment seg = segment(cacheId, pageId);
-
- FullPageId fullId = new FullPageId(pageId, cacheId);
-
- seg.writeLock().lock();
-
- boolean isTrackingPage = trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
-
- try {
- long relPtr = seg.borrowOrAllocateFreePage(pageId);
-
- if (relPtr == INVALID_REL_PTR)
- relPtr = seg.evictPage();
-
- long absPtr = seg.absolute(relPtr);
-
- GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
-
- PageHeader.fullPageId(absPtr, fullId);
- PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
- rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
-
- assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
-
- assert !PageHeader.isAcquired(absPtr) :
- "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) +
- ", absPtr=" + U.hexLong(absPtr) + ']';
-
- setDirty(fullId, absPtr, true, true);
-
- if (isTrackingPage) {
- long pageAddr = absPtr + PAGE_OVERHEAD;
-
- // We are inside segment write lock, so no other thread can pin this tracking page yet.
- // We can modify page buffer directly.
- if (PageIO.getType(pageAddr) == 0) {
- trackingIO.initNewPage(pageAddr, pageId, pageSize());
-
- if (!sharedCtx.wal().isAlwaysWriteFullPages())
- sharedCtx.wal().log(
- new InitNewPageRecord(
- cacheId,
- pageId,
- trackingIO.getType(),
- trackingIO.getVersion(), pageId
- )
- );
- else
- sharedCtx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize()));
- }
- }
-
- seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partTag(cacheId, partId));
- }
- finally {
- seg.writeLock().unlock();
- }
-
- //we have allocated 'tracking' page, we need to allocate regular one
- return isTrackingPage ? allocatePage(cacheId, partId, flags) : pageId;
- }
-
- /** {@inheritDoc} */
- @Override public ByteBuffer pageBuffer(long pageAddr) {
- return wrapPointer(pageAddr, pageSize());
- }
-
- /** {@inheritDoc} */
- @Override public boolean freePage(int cacheId, long pageId) throws IgniteCheckedException {
- assert false : "Free page should be never called directly when persistence is enabled.";
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public long metaPageId(int cacheId) throws IgniteCheckedException {
- return storeMgr.metaPageId(cacheId);
- }
-
- /** {@inheritDoc} */
- @Override public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException {
- return PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
- }
-
- /** {@inheritDoc} */
- @Override public long acquirePage(int cacheId, long pageId) throws IgniteCheckedException {
- return acquirePage(cacheId, pageId, false);
- }
-
- /** {@inheritDoc} */
- @Override public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException {
- FullPageId fullId = new FullPageId(pageId, cacheId);
-
- int partId = PageIdUtils.partId(pageId);
-
- Segment seg = segment(cacheId, pageId);
-
- seg.readLock().lock();
-
- try {
- long relPtr = seg.loadedPages.get(
- cacheId,
- PageIdUtils.effectivePageId(pageId),
- seg.partTag(cacheId, partId),
- INVALID_REL_PTR,
- INVALID_REL_PTR
- );
-
- // The page is loaded to the memory.
- if (relPtr != INVALID_REL_PTR) {
- long absPtr = seg.absolute(relPtr);
-
- seg.acquirePage(absPtr);
-
- return absPtr;
- }
- }
- finally {
- seg.readLock().unlock();
- }
-
- seg.writeLock().lock();
-
- try {
- // Double-check.
- long relPtr = seg.loadedPages.get(
- cacheId,
- PageIdUtils.effectivePageId(pageId),
- seg.partTag(cacheId, partId),
- INVALID_REL_PTR,
- OUTDATED_REL_PTR
- );
-
- long absPtr;
-
- if (relPtr == INVALID_REL_PTR) {
- relPtr = seg.borrowOrAllocateFreePage(pageId);
-
- if (relPtr == INVALID_REL_PTR)
- relPtr = seg.evictPage();
-
- absPtr = seg.absolute(relPtr);
-
- PageHeader.fullPageId(absPtr, fullId);
- PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
-
- assert !PageHeader.isAcquired(absPtr) :
- "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) +
- ", absPtr=" + U.hexLong(absPtr) + ']';
-
- // We can clear dirty flag after the page has been allocated.
- setDirty(fullId, absPtr, false, false);
-
- seg.loadedPages.put(
- cacheId,
- PageIdUtils.effectivePageId(pageId),
- relPtr,
- seg.partTag(cacheId, partId)
- );
-
- long pageAddr = absPtr + PAGE_OVERHEAD;
-
- if (!restore) {
- try {
- ByteBuffer buf = wrapPointer(pageAddr, pageSize());
-
- memMetrics.updatePageReplaceRate();
-
- storeMgr.read(cacheId, pageId, buf);
- }
- catch (IgniteDataIntegrityViolationException ignore) {
- U.warn(log, "Failed to read page (data integrity violation encountered, will try to " +
- "restore using existing WAL) [fullPageId=" + fullId + ']');
-
- tryToRestorePage(fullId, absPtr);
-
- seg.acquirePage(absPtr);
-
- return absPtr;
- }
- }
- else {
- GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
-
- // Must init page ID in order to ensure RWLock tag consistency.
- PageIO.setPageId(pageAddr, pageId);
- }
-
- rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
- }
- else if (relPtr == OUTDATED_REL_PTR) {
- assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
-
- relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
-
- absPtr = seg.absolute(relPtr);
-
- long pageAddr = absPtr + PAGE_OVERHEAD;
-
- GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
-
- PageHeader.fullPageId(absPtr, fullId);
- PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
- PageIO.setPageId(pageAddr, pageId);
-
- assert !PageHeader.isAcquired(absPtr) :
- "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) +
- ", absPtr=" + U.hexLong(absPtr) + ']';
-
- rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
- }
- else
- absPtr = seg.absolute(relPtr);
-
- seg.acquirePage(absPtr);
-
- return absPtr;
- }
- finally {
- seg.writeLock().unlock();
- }
- }
-
- /**
- * @param seg Segment.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param rmv {@code True} if page should be removed.
- * @return Relative pointer to refreshed page.
- */
- private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean rmv) {
- assert seg.writeLock().isHeldByCurrentThread();
-
- int tag = seg.partTag(cacheId, PageIdUtils.partId(pageId));
-
- long relPtr = seg.loadedPages.refresh(cacheId, PageIdUtils.effectivePageId(pageId), tag);
-
- long absPtr = seg.absolute(relPtr);
-
- GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
-
- long tmpBufPtr = PageHeader.tempBufferPointer(absPtr);
-
- if (tmpBufPtr != INVALID_REL_PTR) {
- GridUnsafe.setMemory(checkpointPool.absolute(tmpBufPtr) + PAGE_OVERHEAD, pageSize(), (byte)0);
-
- PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
- PageHeader.dirty(absPtr, false);
- PageHeader.tempDirty(absPtr, false);
-
- // We pinned the page when allocated the temp buffer, release it now.
- PageHeader.releasePage(absPtr);
-
- checkpointPool.releaseFreePage(tmpBufPtr);
- }
-
- if (rmv)
- seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId), tag);
-
- return relPtr;
- }
-
- /**
- * @param fullId Full id.
- * @param absPtr Absolute pointer.
- */
- private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheckedException {
- Long tmpAddr = null;
-
- try {
- ByteBuffer curPage = null;
- ByteBuffer lastValidPage = null;
-
- for (IgniteBiTuple<WALPointer, WALRecord> tuple : walMgr.replay(null)) {
- switch (tuple.getValue().type()) {
- case PAGE_RECORD:
- PageSnapshot snapshot = (PageSnapshot)tuple.getValue();
-
- if (snapshot.fullPageId().equals(fullId)) {
- if (tmpAddr == null) {
- assert snapshot.pageData().length <= pageSize() : snapshot.pageData().length;
-
- tmpAddr = GridUnsafe.allocateMemory(pageSize());
- }
-
- if (curPage == null)
- curPage = wrapPointer(tmpAddr, pageSize());
-
- PageUtils.putBytes(tmpAddr, 0, snapshot.pageData());
- }
-
- break;
-
- case CHECKPOINT_RECORD:
- CheckpointRecord rec = (CheckpointRecord)tuple.getValue();
-
- assert !rec.end();
-
- if (curPage != null) {
- lastValidPage = curPage;
- curPage = null;
- }
-
- break;
-
- case MEMORY_RECOVERY: // It means that previous checkpoint was broken.
- curPage = null;
-
- break;
-
- default:
- if (tuple.getValue() instanceof PageDeltaRecord) {
- PageDeltaRecord deltaRecord = (PageDeltaRecord)tuple.getValue();
-
- if (curPage != null
- && deltaRecord.pageId() == fullId.pageId()
- && deltaRecord.cacheId() == fullId.cacheId()) {
- assert tmpAddr != null;
-
- deltaRecord.applyDelta(this, tmpAddr);
- }
- }
- }
- }
-
- ByteBuffer restored = curPage == null ? lastValidPage : curPage;
-
- if (restored == null)
- throw new AssertionError(String.format(
- "Page is broken. Can't restore it from WAL. (cacheId = %d, pageId = %X).",
- fullId.cacheId(), fullId.pageId()
- ));
-
- long pageAddr = writeLockPage(absPtr, fullId, false);
-
- try {
- pageBuffer(pageAddr).put(restored);
- }
- finally {
- writeUnlockPage(absPtr, fullId, null, true, false);
- }
- }
- finally {
- if (tmpAddr != null)
- GridUnsafe.freeMemory(tmpAddr);
- }
- }
-
- /** {@inheritDoc} */
- @Override public int pageSize() {
- return sysPageSize - PAGE_OVERHEAD;
- }
-
- /** {@inheritDoc} */
- @Override public int systemPageSize() {
- return sysPageSize;
- }
-
- /** {@inheritDoc} */
- @Override public boolean safeToUpdate() {
- for (Segment segment : segments)
- if (!segment.safeToUpdate())
- return false;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException {
- Collection[] collections = new Collection[segments.length];
-
- for (int i = 0; i < segments.length; i++) {
- Segment seg = segments[i];
-
- if (seg.segCheckpointPages != null)
- throw new IgniteException("Failed to begin checkpoint (it is already in progress).");
-
- collections[i] = seg.segCheckpointPages = seg.dirtyPages;
-
- seg.dirtyPages = new GridConcurrentHashSet<>();
- }
-
- memMetrics.resetDirtyPages();
-
- return new GridMultiCollectionWrapper<>(collections);
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings({"unchecked", "TooBroadScope"})
- @Override public void finishCheckpoint() {
- for (Segment seg : segments)
- seg.segCheckpointPages = null;
- }
-
- /** {@inheritDoc} */
- @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) {
- assert tmpBuf.remaining() == pageSize();
-
- Segment seg = segment(fullId.cacheId(), fullId.pageId());
-
- long absPtr = 0;
-
- long relPtr;
-
- int tag;
-
- seg.readLock().lock();
-
- try {
- tag = seg.partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId()));
-
- relPtr = seg.loadedPages.get(
- fullId.cacheId(),
- PageIdUtils.effectivePageId(fullId.pageId()),
- tag,
- INVALID_REL_PTR,
- OUTDATED_REL_PTR
- );
-
- // Page may have been cleared during eviction. We have nothing to do in this case.
- if (relPtr == INVALID_REL_PTR)
- return null;
-
- if (relPtr != OUTDATED_REL_PTR) {
- absPtr = seg.absolute(relPtr);
-
- // Pin the page until page will not be copied.
- if (PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR)
- PageHeader.acquirePage(absPtr);
- }
- }
- finally {
- seg.readLock().unlock();
- }
-
- if (relPtr == OUTDATED_REL_PTR) {
- seg.writeLock().lock();
-
- try {
- // Double-check.
- relPtr = seg.loadedPages.get(
- fullId.cacheId(),
- PageIdUtils.effectivePageId(fullId.pageId()),
- seg.partTag(
- fullId.cacheId(),
- PageIdUtils.partId(fullId.pageId())
- ),
- INVALID_REL_PTR,
- OUTDATED_REL_PTR
- );
-
- if (relPtr == INVALID_REL_PTR)
- return null;
-
- if (relPtr == OUTDATED_REL_PTR) {
- relPtr = refreshOutdatedPage(
- seg,
- fullId.cacheId(),
- PageIdUtils.effectivePageId(fullId.pageId()),
- true
- );
-
- seg.pool.releaseFreePage(relPtr);
- }
-
- return null;
- }
- finally {
- seg.writeLock().unlock();
- }
-
- }
- else {
- copyPageForCheckpoint(absPtr, fullId, tmpBuf, tracker);
-
- return tag;
- }
- }
-
- /**
- * @param absPtr Absolute ptr.
- * @param fullId Full id.
- * @param tmpBuf Tmp buffer.
- */
- private void copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) {
- assert absPtr != 0;
-
- long tmpRelPtr;
-
- rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
-
- try {
- tmpRelPtr = PageHeader.tempBufferPointer(absPtr);
-
- clearCheckpoint(fullId);
-
- if (tmpRelPtr != INVALID_REL_PTR)
- PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
- else {
- copyInBuffer(absPtr, tmpBuf);
-
- PageHeader.dirty(absPtr, false);
-
- PageHeader.releasePage(absPtr);
-
- return;
- }
- }
- finally {
- rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
- }
-
- assert tmpRelPtr != 0;
-
- long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
-
- copyInBuffer(tmpAbsPtr, tmpBuf);
-
- GridUnsafe.setMemory(tmpAbsPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
-
- PageHeader.dirty(tmpAbsPtr, false);
-
- if (tracker != null)
- tracker.onCowPageWritten();
-
- checkpointPool.releaseFreePage(tmpRelPtr);
-
- // We pinned the page when allocated the temp buffer, release it now.
- PageHeader.releasePage(absPtr);
- }
-
- /**
- * @param absPtr Absolute ptr.
- * @param tmpBuf Tmp buffer.
- */
- private void copyInBuffer(long absPtr, ByteBuffer tmpBuf) {
- if (tmpBuf.isDirect()) {
- long tmpPtr = ((DirectBuffer)tmpBuf).address();
-
- GridUnsafe.copyMemory(absPtr + PAGE_OVERHEAD, tmpPtr, pageSize());
-
- assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
- assert GridUnsafe.getInt(tmpPtr + 4) == 0; //TODO GG-11480
- }
- else {
- byte[] arr = tmpBuf.array();
-
- assert arr != null;
- assert arr.length == pageSize();
-
- GridUnsafe.copyMemory(null, absPtr + PAGE_OVERHEAD, arr, GridUnsafe.BYTE_ARR_OFF, pageSize());
- }
- }
-
- /** {@inheritDoc} */
- @Override public int invalidate(int cacheId, int partId) {
- int tag = 0;
-
- for (Segment seg : segments) {
- seg.writeLock().lock();
-
- try {
- int newTag = seg.incrementPartTag(cacheId, partId);
-
- if (tag == 0)
- tag = newTag;
-
- assert tag == newTag;
- }
- finally {
- seg.writeLock().unlock();
- }
- }
-
- return tag;
- }
-
- /** {@inheritDoc} */
- @Override public void onCacheGroupDestroyed(int grpId) {
- for (Segment seg : segments) {
- seg.writeLock().lock();
-
- try {
- seg.resetPartTags(grpId);
- }
- finally {
- seg.writeLock().unlock();
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgniteInternalFuture<Void> clearAsync(
- GridPredicate3<Integer, Long, Integer> pred,
- boolean cleanDirty
- ) {
- CountDownFuture completeFut = new CountDownFuture(segments.length);
-
- for (Segment seg : segments) {
- Runnable clear = new ClearSegmentRunnable(seg, pred, cleanDirty, completeFut, pageSize());
-
- try {
- asyncRunner.execute(clear);
- }
- catch (RejectedExecutionException ignore) {
- clear.run();
- }
- }
-
- return completeFut;
- }
-
- /** {@inheritDoc} */
- @Override public long loadedPages() {
- long total = 0;
-
- for (Segment seg : segments) {
- seg.readLock().lock();
-
- try {
- total += seg.loadedPages.size();
- }
- finally {
- seg.readLock().unlock();
- }
- }
-
- return total;
- }
-
- /**
- * @return Total number of acquired pages.
- */
- public long acquiredPages() {
- long total = 0;
-
- for (Segment seg : segments) {
- seg.readLock().lock();
-
- try {
- total += seg.acquiredPages();
- }
- finally {
- seg.readLock().unlock();
- }
- }
-
- return total;
- }
-
- /**
- * @param absPtr Absolute pointer to read lock.
- * @param force Force flag.
- * @return Pointer to the page read buffer.
- */
- private long readLockPage(long absPtr, FullPageId fullId, boolean force) {
- int tag = force ? -1 : PageIdUtils.tag(fullId.pageId());
-
- boolean locked = rwLock.readLock(absPtr + PAGE_LOCK_OFFSET, tag);
-
- if (!locked)
- return 0;
-
- PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
-
- assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
-
- return absPtr + PAGE_OVERHEAD;
- }
-
- /** {@inheritDoc} */
- @Override public long readLockForce(int cacheId, long pageId, long page) {
- return readLockPage(page, new FullPageId(pageId, cacheId), true);
- }
-
- /**
- * @param absPtr Absolute pointer to unlock.
- */
- void readUnlockPage(long absPtr) {
- rwLock.readUnlock(absPtr + PAGE_LOCK_OFFSET);
- }
-
- /**
- * Checks if a page has temp copy buffer.
- *
- * @param absPtr Absolute pointer.
- * @return {@code True} if a page has temp buffer.
- */
- public boolean hasTempCopy(long absPtr) {
- return PageHeader.tempBufferPointer(absPtr) != INVALID_REL_PTR;
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @return Pointer to the page write buffer or {@code 0} if page was not locked.
- */
- long tryWriteLockPage(long absPtr, FullPageId fullId, boolean checkTag) {
- int tag = checkTag ? PageIdUtils.tag(fullId.pageId()) : OffheapReadWriteLock.TAG_LOCK_ALWAYS;
-
- if (!rwLock.tryWriteLock(absPtr + PAGE_LOCK_OFFSET, tag))
- return 0;
-
- return postWriteLockPage(absPtr, fullId);
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @return Pointer to the page write buffer.
- */
- private long writeLockPage(long absPtr, FullPageId fullId, boolean checkTag) {
- int tag = checkTag ? PageIdUtils.tag(fullId.pageId()) : OffheapReadWriteLock.TAG_LOCK_ALWAYS;
-
- boolean locked = rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, tag);
-
- return locked ? postWriteLockPage(absPtr, fullId) : 0;
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @return Pointer to the page write buffer.
- */
- private long postWriteLockPage(long absPtr, FullPageId fullId) {
- PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
-
- // Create a buffer copy if the page is scheduled for a checkpoint.
- if (isInCheckpoint(fullId) && PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR) {
- long tmpRelPtr = checkpointPool.borrowOrAllocateFreePage(fullId.pageId());
-
- if (tmpRelPtr == INVALID_REL_PTR) {
- rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
-
- throw new IgniteException(
- "Failed to allocate temporary buffer for checkpoint " +
- "(increase checkpointPageBufferSize configuration property)");
- }
-
- // Pin the page until checkpoint is not finished.
- PageHeader.acquirePage(absPtr);
-
- long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
-
- assert !PageHeader.tempDirty(tmpAbsPtr);
-
- GridUnsafe.copyMemory(
- null,
- absPtr + PAGE_OVERHEAD,
- null,
- tmpAbsPtr + PAGE_OVERHEAD,
- pageSize()
- );
-
- PageHeader.dirty(absPtr, false);
- PageHeader.tempBufferPointer(absPtr, tmpRelPtr);
-
- assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
- assert GridUnsafe.getInt(tmpAbsPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
- }
-
- assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
-
- return absPtr + PAGE_OVERHEAD;
- }
-
- /**
- * @param page Page pointer.
- * @param walPlc Full page WAL record policy.
- */
- private void writeUnlockPage(
- long page,
- FullPageId fullId,
- Boolean walPlc,
- boolean markDirty,
- boolean restore
- ) {
- boolean dirty = isDirty(page);
-
- //if page is for restore, we shouldn't mark it as changed
- if (!restore && markDirty && !dirty)
- changeTracker.apply(page, fullId, this);
-
- boolean pageWalRec = markDirty && walPlc != FALSE && (walPlc == TRUE || !dirty);
-
- assert GridUnsafe.getInt(page + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
-
- if (markDirty)
- setDirty(fullId, page, markDirty, false);
-
- beforeReleaseWrite(fullId, page + PAGE_OVERHEAD, pageWalRec);
-
- long pageId = PageIO.getPageId(page + PAGE_OVERHEAD);
-
- try {
- rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
- }
- catch (AssertionError ex) {
- StringBuilder sb = new StringBuilder(sysPageSize * 2);
-
- for (int i = 0; i < systemPageSize(); i += 8)
- sb.append(U.hexLong(GridUnsafe.getLong(page + i)));
-
- U.error(log, "Failed to unlock page [fullPageId=" + fullId + ", binPage=" + sb + ']');
-
- throw ex;
- }
- }
-
- /**
- * @param absPtr Absolute pointer to the page.
- * @return {@code True} if write lock acquired for the page.
- */
- boolean isPageWriteLocked(long absPtr) {
- return rwLock.isWriteLocked(absPtr + PAGE_LOCK_OFFSET);
- }
-
- /**
- * @param absPtr Absolute pointer to the page.
- * @return {@code True} if read lock acquired for the page.
- */
- boolean isPageReadLocked(long absPtr) {
- return rwLock.isReadLocked(absPtr + PAGE_LOCK_OFFSET);
- }
-
- /**
- * @param ptr Pointer to wrap.
- * @param len Memory location length.
- * @return Wrapped buffer.
- */
- ByteBuffer wrapPointer(long ptr, int len) {
- ByteBuffer buf = nioAccess.newDirectByteBuffer(ptr, len, null);
-
- buf.order(ByteOrder.nativeOrder());
-
- return buf;
- }
-
- /**
- * @param pageId Page ID to check if it was added to the checkpoint list.
- * @return {@code True} if it was added to the checkpoint list.
- */
- boolean isInCheckpoint(FullPageId pageId) {
- Segment seg = segment(pageId.cacheId(), pageId.pageId());
-
- Collection<FullPageId> pages0 = seg.segCheckpointPages;
-
- return pages0 != null && pages0.contains(pageId);
- }
-
- /**
- * @param fullPageId Page ID to clear.
- */
- void clearCheckpoint(FullPageId fullPageId) {
- Segment seg = segment(fullPageId.cacheId(), fullPageId.pageId());
-
- Collection<FullPageId> pages0 = seg.segCheckpointPages;
-
- assert pages0 != null;
-
- pages0.remove(fullPageId);
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @return {@code True} if page is dirty.
- */
- boolean isDirty(long absPtr) {
- return PageHeader.dirty(absPtr);
- }
-
- /**
- * Gets the number of active pages across all segments. Used for test purposes only.
- *
- * @return Number of active pages.
- */
- public int activePagesCount() {
- int total = 0;
-
- for (Segment seg : segments)
- total += seg.acquiredPages();
-
- return total;
- }
-
- /**
- * This method must be called in synchronized context.
- *
- * @param absPtr Absolute pointer.
- * @param dirty {@code True} dirty flag.
- * @param forceAdd If this flag is {@code true}, then the page will be added to the dirty set regardless whether the
- * old flag was dirty or not.
- */
- void setDirty(FullPageId pageId, long absPtr, boolean dirty, boolean forceAdd) {
- boolean wasDirty = PageHeader.dirty(absPtr, dirty);
-
- if (dirty) {
- if (!wasDirty || forceAdd) {
- boolean added = segment(pageId.cacheId(), pageId.pageId()).dirtyPages.add(pageId);
-
- if (added)
- memMetrics.incrementDirtyPages();
- }
- }
- else {
- boolean rmv = segment(pageId.cacheId(), pageId.pageId()).dirtyPages.remove(pageId);
-
- if (rmv)
- memMetrics.decrementDirtyPages();
- }
- }
-
- /**
- *
- */
- void beforeReleaseWrite(FullPageId pageId, long ptr, boolean pageWalRec) {
- if (walMgr != null && (pageWalRec || walMgr.isAlwaysWriteFullPages())) {
- try {
- walMgr.log(new PageSnapshot(pageId, ptr, pageSize()));
- }
- catch (IgniteCheckedException e) {
- // TODO ignite-db.
- throw new IgniteException(e);
- }
- }
- }
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return Segment.
- */
- private Segment segment(int cacheId, long pageId) {
- int idx = segmentIndex(cacheId, pageId, segments.length);
-
- return segments[idx];
- }
-
- /**
- * @param pageId Page ID.
- * @return Segment index.
- */
- public static int segmentIndex(int cacheId, long pageId, int segments) {
- pageId = PageIdUtils.effectivePageId(pageId);
-
- // Take a prime number larger than total number of partitions.
- int hash = U.hash(pageId * 65537 + cacheId);
-
- return U.safeAbs(hash) % segments;
- }
-
- /**
- *
- */
- private class PagePool {
- /** Segment index. */
- protected final int idx;
-
- /** Direct memory region. */
- protected final DirectMemoryRegion region;
-
- /** */
- protected long lastAllocatedIdxPtr;
-
- /** Pointer to the address of the free page list. */
- protected long freePageListPtr;
-
- /** Pages base. */
- protected long pagesBase;
-
- /**
- * @param idx Index.
- * @param region Region
- */
- protected PagePool(int idx, DirectMemoryRegion region) {
- this.idx = idx;
- this.region = region;
-
- long base = (region.address() + 7) & ~0x7;
-
- freePageListPtr = base;
-
- base += 8;
-
- lastAllocatedIdxPtr = base;
-
- base += 8;
-
- // Align page start by
- pagesBase = base;
-
- GridUnsafe.putLong(freePageListPtr, INVALID_REL_PTR);
- GridUnsafe.putLong(lastAllocatedIdxPtr, 1L);
- }
-
- /**
- * Allocates a new free page.
- *
- * @param pageId Page ID to to initialize.
- * @return Relative pointer to the allocated page.
- * @throws GridOffHeapOutOfMemoryException If failed to allocate new free page.
- */
- private long borrowOrAllocateFreePage(long pageId) throws GridOffHeapOutOfMemoryException {
- long relPtr = borrowFreePage();
-
- return relPtr != INVALID_REL_PTR ? relPtr : allocateFreePage(pageId);
- }
-
- /**
- * @return Relative pointer to a free page that was borrowed from the allocated pool.
- */
- private long borrowFreePage() {
- while (true) {
- long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr);
-
- long freePageRelPtr = freePageRelPtrMasked & ADDRESS_MASK;
-
- if (freePageRelPtr != INVALID_REL_PTR) {
- long freePageAbsPtr = absolute(freePageRelPtr);
-
- long nextFreePageRelPtr = GridUnsafe.getLong(freePageAbsPtr) & ADDRESS_MASK;
-
- long cnt = ((freePageRelPtrMasked & COUNTER_MASK) + COUNTER_INC) & COUNTER_MASK;
-
- if (GridUnsafe.compareAndSwapLong(null, freePageListPtr, freePageRelPtrMasked, nextFreePageRelPtr | cnt)) {
- GridUnsafe.putLong(freePageAbsPtr, PAGE_MARKER);
-
- return freePageRelPtr;
- }
- }
- else
- return INVALID_REL_PTR;
- }
- }
-
- /**
- * @param pageId Page ID.
- * @return Relative pointer of the allocated page.
- * @throws GridOffHeapOutOfMemoryException If failed to allocate new free page.
- */
- private long allocateFreePage(long pageId) throws GridOffHeapOutOfMemoryException {
- long limit = region.address() + region.size();
-
- while (true) {
- long lastIdx = GridUnsafe.getLong(lastAllocatedIdxPtr);
-
- // Check if we have enough space to allocate a page.
- if (pagesBase + (lastIdx + 1) * sysPageSize > limit)
- return INVALID_REL_PTR;
-
- if (GridUnsafe.compareAndSwapLong(null, lastAllocatedIdxPtr, lastIdx, lastIdx + 1)) {
- long absPtr = pagesBase + lastIdx * sysPageSize;
-
- assert (lastIdx & SEGMENT_INDEX_MASK) == 0L;
-
- long relative = relative(lastIdx);
-
- assert relative != INVALID_REL_PTR;
-
- PageHeader.initNew(absPtr, relative);
-
- rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
-
- return relative;
- }
- }
- }
-
- /**
- * @param relPtr Relative pointer to free.
- */
- private void releaseFreePage(long relPtr) {
- long absPtr = absolute(relPtr);
-
- assert !PageHeader.isAcquired(absPtr) : "Release pinned page: " + PageHeader.fullPageId(absPtr);
-
- while (true) {
- long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr);
-
- long freePageRelPtr = freePageRelPtrMasked & RELATIVE_PTR_MASK;
-
- GridUnsafe.putLong(absPtr, freePageRelPtr);
-
- if (GridUnsafe.compareAndSwapLong(null, freePageListPtr, freePageRelPtrMasked, relPtr))
- return;
- }
- }
-
- /**
- * @param relativePtr Relative pointer.
- * @return Absolute pointer.
- */
- long absolute(long relativePtr) {
- int segIdx = (int)((relativePtr >> 40) & 0xFFFF);
-
- assert segIdx == idx : "expected=" + idx + ", actual=" + segIdx;
-
- long pageIdx = relativePtr & ~SEGMENT_INDEX_MASK;
-
- long off = pageIdx * sysPageSize;
-
- return pagesBase + off;
- }
-
- /**
- * @param pageIdx Page index in the pool.
- * @return Relative pointer.
- */
- private long relative(long pageIdx) {
- return pageIdx | ((long)idx) << 40;
- }
-
- /**
- * @return Max number of pages in the pool.
- */
- private int pages() {
- return (int)((region.size() - (pagesBase - region.address())) / sysPageSize);
- }
- }
-
- /**
- *
- */
- private class Segment extends ReentrantReadWriteLock {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- private static final double FULL_SCAN_THRESHOLD = 0.4;
-
- /** Page ID to relative pointer map. */
- private FullPageIdTable loadedPages;
-
- /** */
- private long acquiredPagesPtr;
-
- /** */
- private PagePool pool;
-
- /** */
- private long memPerTbl;
-
- /** Pages marked as dirty since the last checkpoint. */
- private Collection<FullPageId> dirtyPages = new GridConcurrentHashSet<>();
-
- /** */
- private volatile Collection<FullPageId> segCheckpointPages;
-
- /** */
- private final int maxDirtyPages;
-
- /** */
- private final Map<T2<Integer, Integer>, Integer> partTagMap = new HashMap<>();
-
- /**
- * @param region Memory region.
- */
- private Segment(int idx, DirectMemoryRegion region, int cpPoolPages) {
- long totalMemory = region.size();
-
- int pages = (int)(totalMemory / sysPageSize);
-
- memPerTbl = requiredSegmentTableMemory(pages);
-
- acquiredPagesPtr = region.address();
-
- GridUnsafe.putIntVolatile(null, acquiredPagesPtr, 0);
-
- loadedPages = new FullPageIdTable(region.address() + 8, memPerTbl, true);
-
- DirectMemoryRegion poolRegion = region.slice(memPerTbl + 8);
-
- pool = new PagePool(idx, poolRegion);
-
- maxDirtyPages = Math.min(pool.pages() * 2 / 3, cpPoolPages);
- }
-
- /**
- *
- */
- private boolean safeToUpdate() {
- return dirtyPages.size() < maxDirtyPages;
- }
-
- /**
- * @return Max number of pages this segment can allocate.
- */
- private int pages() {
- return pool.pages();
- }
-
- /**
- * @return Memory allocated for pages table.
- */
- private long tableSize() {
- return memPerTbl;
- }
-
- /**
- * @param absPtr Page absolute address to acquire.
- */
- private void acquirePage(long absPtr) {
- PageHeader.acquirePage(absPtr);
-
- updateAtomicInt(acquiredPagesPtr, 1);
- }
-
- /**
- * @param absPtr Page absolute address to release.
- */
- private void releasePage(long absPtr) {
- PageHeader.releasePage(absPtr);
-
- updateAtomicInt(acquiredPagesPtr, -1);
- }
-
- /**
- * @return Total number of acquired pages.
- */
- private int acquiredPages() {
- return GridUnsafe.getInt(acquiredPagesPtr);
- }
-
- /**
- * @param pageId Page ID.
- * @return Page relative pointer.
- */
- private long borrowOrAllocateFreePage(long pageId) {
- return pool.borrowOrAllocateFreePage(pageId);
- }
-
- /**
- * Prepares a page for eviction, if needed.
- *
- * @param fullPageId Candidate page full ID.
- * @param absPtr Absolute pointer of the page to evict.
- * @return {@code True} if it is ok to evict this page, {@code false} if another page should be selected.
- * @throws IgniteCheckedException If failed to write page to the underlying store during eviction.
- */
- private boolean prepareEvict(FullPageId fullPageId, long absPtr) throws IgniteCheckedException {
- assert writeLock().isHeldByCurrentThread();
-
- // Do not evict cache meta pages.
- if (fullPageId.pageId() == storeMgr.metaPageId(fullPageId.cacheId()))
- return false;
-
- if (PageHeader.isAcquired(absPtr))
- return false;
-
- Collection<FullPageId> cpPages = segCheckpointPages;
-
- if (isDirty(absPtr)) {
- // Can evict a dirty page only if should be written by a checkpoint.
- if (cpPages != null && cpPages.contains(fullPageId)) {
- assert storeMgr != null;
-
- flushDirtyPage.applyx(
- fullPageId,
- wrapPointer(absPtr + PAGE_OVERHEAD, pageSize()),
- partTag(
- fullPageId.cacheId(),
- PageIdUtils.partId(fullPageId.pageId())
- )
- );
-
- setDirty(fullPageId, absPtr, false, true);
-
- cpPages.remove(fullPageId);
-
- return true;
- }
-
- return false;
- }
- else
- // Page was not modified, ok to evict.
- return true;
- }
-
- /**
- * Evict random oldest page from memory to storage.
- *
- * @return Relative address for evicted page.
- * @throws IgniteCheckedException If failed to evict page.
- */
- private long evictPage() throws IgniteCheckedException {
- assert getWriteHoldCount() > 0;
-
- if (!pageEvictWarned) {
- pageEvictWarned = true;
-
- U.warn(log, "Page evictions started, this will affect storage performance (consider increasing " +
- "MemoryConfiguration#setPageCacheSize).");
- }
-
- final ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- final int cap = loadedPages.capacity();
-
- if (acquiredPages() >= loadedPages.size())
- throw new IgniteOutOfMemoryException("Failed to evict page from segment (all pages are acquired).");
-
- // With big number of random picked pages we may fall into infinite loop, because
- // every time the same page may be found.
- Set<Long> ignored = null;
-
- long relEvictAddr = INVALID_REL_PTR;
-
- int iterations = 0;
-
- while (true) {
- long cleanAddr = INVALID_REL_PTR;
- long cleanTs = Long.MAX_VALUE;
- long dirtyTs = Long.MAX_VALUE;
- long dirtyAddr = INVALID_REL_PTR;
-
- for (int i = 0; i < RANDOM_PAGES_EVICT_NUM; i++) {
- ++iterations;
-
- if (iterations > pool.pages() * FULL_SCAN_THRESHOLD)
- break;
-
- // We need to lookup for pages only in current segment for thread safety,
- // so peeking random memory will lead to checking for found page segment.
- // It's much faster to check available pages for segment right away.
- EvictCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
-
- assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
-
- long rndAddr = nearest.relativePointer();
-
- int partTag = nearest.tag();
-
- final long absPageAddr = absolute(rndAddr);
-
- FullPageId fullId = PageHeader.fullPageId(absPageAddr);
-
- // Check page mapping consistency.
- assert fullId.equals(nearest.fullId()) : "Invalid page mapping [tableId=" + nearest.fullId() +
- ", actual=" + fullId + ", nearest=" + nearest;
-
- boolean outdated = partTag < partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId()));
-
- if (outdated)
- return refreshOutdatedPage(this, fullId.cacheId(), fullId.pageId(), true);
-
- boolean pinned = PageHeader.isAcquired(absPageAddr);
-
- boolean skip = ignored != null && ignored.contains(rndAddr);
-
- if (relEvictAddr == rndAddr || pinned || skip) {
- i--;
-
- continue;
- }
-
- final long pageTs = PageHeader.readTimestamp(absPageAddr);
-
- final boolean dirty = isDirty(absPageAddr);
-
- if (pageTs < cleanTs && !dirty) {
- cleanAddr = rndAddr;
-
- cleanTs = pageTs;
- }
- else if (pageTs < dirtyTs && dirty) {
- dirtyAddr = rndAddr;
-
- dirtyTs = pageTs;
- }
-
- relEvictAddr = cleanAddr == INVALID_REL_PTR ? dirtyAddr : cleanAddr;
- }
-
- assert relEvictAddr != INVALID_REL_PTR;
-
- final long absEvictAddr = absolute(relEvictAddr);
-
- final FullPageId fullPageId = PageHeader.fullPageId(absEvictAddr);
-
- if (!prepareEvict(fullPageId, absEvictAddr)) {
- if (iterations > 10) {
- if (ignored == null)
- ignored = new HashSet<>();
-
- ignored.add(relEvictAddr);
- }
-
- if (iterations > pool.pages() * FULL_SCAN_THRESHOLD)
- return tryToFindSequentially(cap);
-
- continue;
- }
-
- loadedPages.remove(
- fullPageId.cacheId(),
- PageIdUtils.effectivePageId(fullPageId.pageId()),
- partTag(
- fullPageId.cacheId(),
- PageIdUtils.partId(fullPageId.pageId())
- )
- );
-
- return relEvictAddr;
- }
- }
-
- /**
- * Will scan all segment pages to find one to evict it
- *
- * @param cap Capacity.
- */
- private long tryToFindSequentially(int cap) throws IgniteCheckedException {
- assert getWriteHoldCount() > 0;
-
- long prevAddr = INVALID_REL_PTR;
- int pinnedCnt = 0;
- int failToPrepare = 0;
-
- for (int i = 0; i < cap; i++) {
- final EvictCandidate nearest = loadedPages.getNearestAt(i, INVALID_REL_PTR);
-
- assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
-
- final long addr = nearest.relativePointer();
-
- int partTag = nearest.tag();
-
- final long absPageAddr = absolute(addr);
-
- FullPageId fullId = PageHeader.fullPageId(absPageAddr);
-
- if (partTag < partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId())))
- return refreshOutdatedPage(this, fullId.cacheId(), fullId.pageId(), true);
-
- boolean pinned = PageHeader.isAcquired(absPageAddr);
-
- if (pinned)
- pinnedCnt++;
-
- if (addr == prevAddr || pinned)
- continue;
-
- final long absEvictAddr = absolute(addr);
-
- final FullPageId fullPageId = PageHeader.fullPageId(absEvictAddr);
-
- if (prepareEvict(fullPageId, absEvictAddr)) {
- loadedPages.remove(
- fullPageId.cacheId(),
- PageIdUtils.effectivePageId(fullPageId.pageId()),
- partTag(
- fullPageId.cacheId(),
- PageIdUtils.partId(fullPageId.pageId())
- )
- );
-
- return addr;
- }
- else
- failToPrepare++;
-
- prevAddr = addr;
- }
-
- throw new IgniteOutOfMemoryException("Failed to find a page for eviction [segmentCapacity=" + cap +
- ", loaded=" + loadedPages.size() +
- ", maxDirtyPages=" + maxDirtyPages +
- ", dirtyPages=" + dirtyPages.size() +
- ", cpPages=" + (segCheckpointPages == null ? 0 : segCheckpointPages.size()) +
- ", pinnedInSegment=" + pinnedCnt +
- ", failedToPrepare=" + failToPrepare +
- ']');
- }
-
- /**
- * Delegate to the corresponding page pool.
- *
- * @param relPtr Relative pointer.
- * @return Absolute pointer.
- */
- private long absolute(long relPtr) {
- return pool.absolute(relPtr);
- }
-
- /**
- * @param cacheId Cache ID.
- * @param partId Partition ID.
- * @return Partition tag.
- */
- private int partTag(int cacheId, int partId) {
- assert getReadHoldCount() > 0 || getWriteHoldCount() > 0;
-
- Integer tag = partTagMap.get(new T2<>(cacheId, partId));
-
- if (tag == null)
- return 1;
-
- return tag;
- }
-
- /**
- * @param cacheId Cache ID.
- * @param partId Partition ID.
- * @return New partition tag.
- */
- private int incrementPartTag(int cacheId, int partId) {
- assert getWriteHoldCount() > 0;
-
- T2<Integer, Integer> t = new T2<>(cacheId, partId);
-
- Integer tag = partTagMap.get(t);
-
- if (tag == null) {
- partTagMap.put(t, 2);
-
- return 2;
- }
- else if (tag == Integer.MAX_VALUE) {
- U.warn(log, "Partition tag overflow [cacheId=" + cacheId + ", partId=" + partId + "]");
-
- partTagMap.put(t, 0);
-
- return 0;
- }
- else {
- partTagMap.put(t, tag + 1);
-
- return tag + 1;
- }
- }
-
- /**
- * @param grpId Cache group id.
- */
- private void resetPartTags(int grpId) {
- assert getWriteHoldCount() > 0;
-
- Iterator<T2<Integer, Integer>> iter = partTagMap.keySet().iterator();
-
- while (iter.hasNext()) {
- T2<Integer, Integer> t = iter.next();
-
- if (t.get1() == grpId)
- iter.remove();
- }
- }
- }
-
- /**
- * Gets an estimate for the amount of memory required to store the given number of page IDs
- * in a segment table.
- *
- * @param pages Number of pages to store.
- * @return Memory size estimate.
- */
- private static long requiredSegmentTableMemory(int pages) {
- return FullPageIdTable.requiredMemory(pages) + 8;
- }
-
- /**
- * @param ptr Pointer to update.
- * @param delta Delta.
- */
- private static int updateAtomicInt(long ptr, int delta) {
- while (true) {
- int old = GridUnsafe.getInt(ptr);
-
- int updated = old + delta;
-
- if (GridUnsafe.compareAndSwapInt(null, ptr, old, updated))
- return updated;
- }
- }
-
- /**
- * @param ptr Pointer to update.
- * @param delta Delta.
- */
- private static long updateAtomicLong(long ptr, long delta) {
- while (true) {
- long old = GridUnsafe.getLong(ptr);
-
- long updated = old + delta;
-
- if (GridUnsafe.compareAndSwapLong(null, ptr, old, updated))
- return updated;
- }
- }
-
- /**
- *
- */
- private static class PageHeader {
- /**
- * @param absPtr Absolute pointer to initialize.
- * @param relative Relative pointer to write.
- */
- private static void initNew(long absPtr, long relative) {
- relative(absPtr, relative);
-
- tempBufferPointer(absPtr, INVALID_REL_PTR);
-
- GridUnsafe.putLong(absPtr, PAGE_MARKER);
- GridUnsafe.putInt(absPtr + PAGE_PIN_CNT_OFFSET, 0);
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @return Dirty flag.
- */
- private static boolean dirty(long absPtr) {
- return flag(absPtr, DIRTY_FLAG);
- }
-
- /**
- * @param absPtr Page absolute pointer.
- * @param dirty Dirty flag.
- * @return Previous value of dirty flag.
- */
- private static boolean dirty(long absPtr, boolean dirty) {
- return flag(absPtr, DIRTY_FLAG, dirty);
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @return Dirty flag.
- */
- private static boolean tempDirty(long absPtr) {
- return flag(absPtr, TMP_DIRTY_FLAG);
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @param tmpDirty Temp dirty flag.
- * @return Previous value of temp dirty flag.
- */
- private static boolean tempDirty(long absPtr, boolean tmpDirty) {
- return flag(absPtr, TMP_DIRTY_FLAG, tmpDirty);
- }
-
- /**
- * @param absPtr Absolute pointer.
- * @param flag Flag mask.
- * @return Flag value.
- */
- private static boolean flag(long absPtr, long flag) {
- assert (flag & 0xFFFFFFFFFFFFFFL) == 0;
- assert Long.bitCount(flag) == 1;
-
- long relPtrWithFlags = GridUnsafe.getLong(absPtr + RELATIVE_PTR_OFFSET);
-
- return (relPtrWithFlags & flag) != 0;
- }
-
- /**
- * Sets flag.
- *
- * @param absPtr Absolute pointer.
- * @param flag Flag mask.
- * @param set New flag value.
- * @return Previous flag value.
- */
- private static boolean flag(long absPtr, long flag, boolean set) {
- assert (flag & 0xFFFFFFFFFFFFFFL) == 0;
- assert Long.bitCount(flag) == 1;
-
- long relPtrWithFlags = GridUnsafe.getLong(absPtr + RELATIVE_PTR_OFFSET);
-
- boolean was = (relPtrWithFlags & flag) != 0;
-
- if (set)
- relPtrWithFlags |= flag;
- else
- relPtrWithFlags &= ~flag;
-
- GridUnsafe.putLong(absPtr + RELATIVE_PTR_OFFSET, relPtrWithFlags);
-
- return was;
- }
-
- /**
- * @param absPtr Page pointer.
- * @return If page is pinned.
- */
- private static boolean isAcquired(long absPtr) {
- return GridUnsafe.getInt(absPtr + PAGE_PIN_CNT_OFFSET) > 0;
- }
-
- /**
- * @param absPtr Absolute pointer.
- */
- private static void acquirePage(long absPtr) {
- updateAtomicInt(absPtr + PAGE_PIN_CNT_OFFSET, 1);
- }
-
- /**
- * @param absPtr Absolute pointer.
- */
- private static int releasePage(long absPtr) {
- return updateAtomicInt(absPtr + PAGE_PIN_CNT_OFFSET, -1);
- }
-
- /**
- * Reads relative pointer from the page at the given absolute position.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @return Relative pointer written to the page.
- */
- private static long readRelative(long absPtr) {
- return GridUnsafe.getLong(absPtr + RELATIVE_PTR_OFFSET) & RELATIVE_PTR_MASK;
- }
-
- /**
- * Writes relative pointer to the page at the given absolute position.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @param relPtr Relative pointer to write.
- */
- private static void relative(long absPtr, long relPtr) {
- GridUnsafe.putLong(absPtr + RELATIVE_PTR_OFFSET, relPtr & RELATIVE_PTR_MASK);
- }
-
- /**
- * Volatile write for current timestamp to page in {@code absAddr} address.
- *
- * @param absPtr Absolute page address.
- */
- private static void writeTimestamp(final long absPtr, long tstamp) {
- tstamp >>= 8;
-
- GridUnsafe.putLongVolatile(null, absPtr, (tstamp << 8) | 0x01);
- }
-
- /**
- * Read for timestamp from page in {@code absAddr} address.
- *
- * @param absPtr Absolute page address.
- * @return Timestamp.
- */
- private static long readTimestamp(final long absPtr) {
- long markerAndTs = GridUnsafe.getLong(absPtr);
-
- // Clear last byte as it is occupied by page marker.
- return markerAndTs & ~0xFF;
- }
-
- /**
- * @param absPtr Page absolute pointer.
- * @param tmpRelPtr Temp buffer relative pointer.
- */
- private static void tempBufferPointer(long absPtr, long tmpRelPtr) {
- GridUnsafe.putLong(absPtr + PAGE_TMP_BUF_OFFSET, tmpRelPtr);
- }
-
- /**
- * @param absPtr Page absolute pointer.
- * @return Temp buffer relative pointer.
- */
- private static long tempBufferPointer(long absPtr) {
- return GridUnsafe.getLong(absPtr + PAGE_TMP_BUF_OFFSET);
- }
-
- /**
- * Reads page ID from the page at the given absolute position.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @return Page ID written to the page.
- */
- private static long readPageId(long absPtr) {
- return GridUnsafe.getLong(absPtr + PAGE_ID_OFFSET);
- }
-
- /**
- * Writes page ID to the page at the given absolute position.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @param pageId Page ID to write.
- */
- private static void pageId(long absPtr, long pageId) {
- GridUnsafe.putLong(absPtr + PAGE_ID_OFFSET, pageId);
- }
-
- /**
- * Reads cache ID from the page at the given absolute pointer.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @return Cache ID written to the page.
- */
- private static int readPageCacheId(final long absPtr) {
- return GridUnsafe.getInt(absPtr + PAGE_CACHE_ID_OFFSET);
- }
-
- /**
- * Writes cache ID from the page at the given absolute pointer.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @param cacheId Cache ID to write.
- */
- private static void pageCacheId(final long absPtr, final int cacheId) {
- GridUnsafe.putInt(absPtr + PAGE_CACHE_ID_OFFSET, cacheId);
- }
-
- /**
- * Reads page ID and cache ID from the page at the given absolute pointer.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @return Full page ID written to the page.
- */
- private static FullPageId fullPageId(final long absPtr) {
- return new FullPageId(readPageId(absPtr), readPageCacheId(absPtr));
- }
-
- /**
- * Writes page ID and cache ID from the page at the given absolute pointer.
- *
- * @param absPtr Absolute memory pointer to the page header.
- * @param fullPageId Full page ID to write.
- */
- private static void fullPageId(final long absPtr, final FullPageId fullPageId) {
- pageId(absPtr, fullPageId.pageId());
-
- pageCacheId(absPtr, fullPageId.cacheId());
- }
- }
-
- /**
- *
- */
- private static class ClearSegmentRunnable implements Runnable {
- /** */
- private Segment seg;
-
- /** */
- private GridPredicate3<Integer, Long, Integer> clearPred;
-
- /** */
- private CountDownFuture doneFut;
-
- /** */
- private int pageSize;
-
- /** */
- private boolean rmvDirty;
-
- /**
- * @param seg Segment.
- * @param clearPred Clear predicate.
- * @param doneFut Completion future.
- */
- private ClearSegmentRunnable(
- Segment seg,
- GridPredicate3<Integer, Long, Integer> clearPred,
- boolean rmvDirty,
- CountDownFuture doneFut,
- int pageSize
- ) {
- this.seg = seg;
- this.clearPred = clearPred;
- this.rmvDirty = rmvDirty;
- this.doneFut = doneFut;
- this.pageSize = pageSize;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- int cap = seg.loadedPages.capacity();
-
- int chunkSize = 1000;
- GridLongList ptrs = new GridLongList(chunkSize);
-
- try {
- for (int base = 0; base < cap; ) {
- int boundary = Math.min(cap, base + chunkSize);
-
- seg.writeLock().lock();
-
- try {
- while (base < boundary) {
- long ptr = seg.loadedPages.clearAt(base, clearPred, INVALID_REL_PTR);
-
- if (ptr != INVALID_REL_PTR)
- ptrs.add(ptr);
-
- base++;
- }
- }
- finally {
- seg.writeLock().unlock();
- }
-
- for (int i = 0; i < ptrs.size(); i++) {
- long relPtr = ptrs.get(i);
-
- long absPtr = seg.pool.absolute(relPtr);
-
- if (rmvDirty) {
- FullPageId fullId = PageHeader.fullPageId(absPtr);
-
- seg.dirtyPages.remove(fullId);
- }
-
- GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize, (byte)0);
-
- seg.pool.releaseFreePage(relPtr);
- }
-
- ptrs.clear();
- }
-
- doneFut.onDone((Void)null);
- }
- catch (Throwable e) {
- doneFut.onDone(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInput.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInput.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInput.java
deleted file mode 100644
index 220f27b..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInput.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-
-import java.io.DataInput;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-/**
- * ByteBuffer backed data input
- */
-public interface ByteBufferBackedDataInput extends DataInput {
- /**
- * @return ByteBuffer hold by data input
- */
- public ByteBuffer buffer();
-
- /**
- * ensure that requested count of byte is available in data input and will try to load data if not
- * @param requested Requested number of bytes.
- * @throws IOException If failed.
- */
- public void ensure(int requested) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInputImpl.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInputImpl.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInputImpl.java
deleted file mode 100644
index d5fc9ed..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/ByteBufferBackedDataInputImpl.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * Byte buffer backed data input.
- */
-public class ByteBufferBackedDataInputImpl implements ByteBufferBackedDataInput {
- /** Buffer. */
- private ByteBuffer buf;
-
- /**
- * @param buf New buffer.
- */
- public ByteBufferBackedDataInput buffer(ByteBuffer buf) {
- this.buf = buf;
-
- return this;
- }
-
- /** {@inheritDoc} */
- @Override public ByteBuffer buffer() {
- return buf;
- }
-
- /** {@inheritDoc} */
- @Override public void ensure(int requested) throws IOException {
- if (buf.remaining() < requested)
- throw new IOException("Requested size is greater than buffer: " + requested);
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(@NotNull byte[] b) throws IOException {
- ensure(b.length);
-
- buf.get(b);
- }
-
- /** {@inheritDoc} */
- @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
- ensure(b.length);
-
- buf.get(b, off, len);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int skipBytes(int n) throws IOException {
- ensure(n);
-
- buf.position(buf.position() + n);
-
- return n;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean readBoolean() throws IOException {
- return readByte() == 1;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public byte readByte() throws IOException {
- ensure(1);
-
- return buf.get();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readUnsignedByte() throws IOException {
- return readByte() & 0xFF;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public short readShort() throws IOException {
- ensure(2);
-
- return buf.getShort();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readUnsignedShort() throws IOException {
- return readShort() & 0xFFFF;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public char readChar() throws IOException {
- ensure(2);
-
- return buf.getChar();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readInt() throws IOException {
- ensure(4);
-
- return buf.getInt();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public long readLong() throws IOException {
- ensure(8);
-
- return buf.getLong();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public float readFloat() throws IOException {
- ensure(4);
-
- return buf.getFloat();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public double readDouble() throws IOException {
- ensure(8);
-
- return buf.getDouble();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String readLine() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String readUTF() throws IOException {
- throw new UnsupportedOperationException();
- }
-}
[26/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
new file mode 100644
index 0000000..0ccd3a0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/serializer/RecordV1Serializer.java
@@ -0,0 +1,1654 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.serializer;
+
+import java.io.DataInput;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CacheState;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageSetFreeListPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.FixLeftmostChildRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.FixRemoveId;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InnerReplaceRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MergeRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageAddRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastAllocatedIndex;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulFullSnapshotId;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulSnapshotId;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListInitNewPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListRemovePageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetNextRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.SplitForwardPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.TrackingPageDeltaRecord;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferBackedDataInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.RecordSerializer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.SegmentEofException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.processors.cache.persistence.wal.record.HeaderRecord;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
+
+/**
+ * Record V1 serializer.
+ */
+public class RecordV1Serializer implements RecordSerializer {
+ /** */
+ public static final int HEADER_RECORD_SIZE = /*Type*/1 + /*Pointer */12 + /*Magic*/8 + /*Version*/4 + /*CRC*/4;
+
+ /** */
+ private GridCacheSharedContext cctx;
+
+ /** */
+ private int pageSize;
+
+ /** */
+ private IgniteCacheObjectProcessor co;
+
+ /** */
+ private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
+
+ /**
+ * @param cctx Cache shared context.
+ */
+ public RecordV1Serializer(GridCacheSharedContext cctx) {
+ this.cctx = cctx;
+
+ co = cctx.kernalContext().cacheObjects();
+ pageSize = cctx.database().pageSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int version() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("CastConflictsWithInstanceof")
+ @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException {
+ assert record.size() > 0 && buf.remaining() >= record.size() : record.size();
+
+ int startPos = buf.position();
+
+ buf.put((byte)(record.type().ordinal() + 1));
+
+ putPosition(buf, (FileWALPointer)record.position());
+
+ switch (record.type()) {
+ case PAGE_RECORD:
+ PageSnapshot snap = (PageSnapshot)record;
+
+ buf.putInt(snap.fullPageId().cacheId());
+ buf.putLong(snap.fullPageId().pageId());
+ buf.put(snap.pageData());
+
+ break;
+
+ case MEMORY_RECOVERY:
+ MemoryRecoveryRecord memoryRecoveryRecord = (MemoryRecoveryRecord)record;
+
+ buf.putLong(memoryRecoveryRecord.time());
+
+ break;
+
+ case PARTITION_DESTROY:
+ PartitionDestroyRecord partDestroy = (PartitionDestroyRecord)record;
+
+ buf.putInt(partDestroy.cacheId());
+ buf.putInt(partDestroy.partitionId());
+
+ break;
+
+ case META_PAGE_INIT:
+ MetaPageInitRecord updRootsRec = (MetaPageInitRecord)record;
+
+ buf.putInt(updRootsRec.cacheId());
+ buf.putLong(updRootsRec.pageId());
+
+ buf.putShort((short)updRootsRec.ioType());
+ buf.putShort((short)updRootsRec.ioVersion());
+ buf.putLong(updRootsRec.treeRoot());
+ buf.putLong(updRootsRec.reuseListRoot());
+
+ break;
+
+ case PARTITION_META_PAGE_UPDATE_COUNTERS:
+ MetaPageUpdatePartitionDataRecord partDataRec = (MetaPageUpdatePartitionDataRecord)record;
+
+ buf.putInt(partDataRec.cacheId());
+ buf.putLong(partDataRec.pageId());
+
+ buf.putLong(partDataRec.updateCounter());
+ buf.putLong(partDataRec.globalRemoveId());
+ buf.putInt(partDataRec.partitionSize());
+ buf.putLong(partDataRec.countersPageId());
+ buf.put(partDataRec.state());
+ buf.putInt(partDataRec.allocatedIndexCandidate());
+
+ break;
+
+ case CHECKPOINT_RECORD:
+ CheckpointRecord cpRec = (CheckpointRecord)record;
+
+ assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
+ "Invalid WAL record: " + cpRec;
+
+ FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
+ UUID cpId = cpRec.checkpointId();
+
+ buf.putLong(cpId.getMostSignificantBits());
+ buf.putLong(cpId.getLeastSignificantBits());
+
+ buf.put(walPtr == null ? (byte)0 : 1);
+
+ if (walPtr != null) {
+ buf.putLong(walPtr.index());
+ buf.putInt(walPtr.fileOffset());
+ buf.putInt(walPtr.length());
+ }
+
+ putCacheStates(buf, cpRec.cacheGroupStates());
+
+ buf.put(cpRec.end() ? (byte)1 : 0);
+
+ break;
+
+ case DATA_RECORD:
+ DataRecord dataRec = (DataRecord)record;
+
+ buf.putInt(dataRec.writeEntries().size());
+
+ for (DataEntry dataEntry : dataRec.writeEntries())
+ putDataEntry(buf, dataEntry);
+
+ break;
+
+ case HEADER_RECORD:
+ buf.putLong(HeaderRecord.MAGIC);
+
+ buf.putInt(((HeaderRecord)record).version());
+
+ break;
+
+ case DATA_PAGE_INSERT_RECORD:
+ DataPageInsertRecord diRec = (DataPageInsertRecord)record;
+
+ buf.putInt(diRec.cacheId());
+ buf.putLong(diRec.pageId());
+
+ buf.putShort((short)diRec.payload().length);
+
+ buf.put(diRec.payload());
+
+ break;
+
+ case DATA_PAGE_UPDATE_RECORD:
+ DataPageUpdateRecord uRec = (DataPageUpdateRecord)record;
+
+ buf.putInt(uRec.cacheId());
+ buf.putLong(uRec.pageId());
+ buf.putInt(uRec.itemId());
+
+ buf.putShort((short)uRec.payload().length);
+
+ buf.put(uRec.payload());
+
+ break;
+
+ case DATA_PAGE_INSERT_FRAGMENT_RECORD:
+ final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record;
+
+ buf.putInt(difRec.cacheId());
+ buf.putLong(difRec.pageId());
+
+ buf.putLong(difRec.lastLink());
+ buf.putInt(difRec.payloadSize());
+ buf.put(difRec.payload());
+
+ break;
+
+ case DATA_PAGE_REMOVE_RECORD:
+ DataPageRemoveRecord drRec = (DataPageRemoveRecord)record;
+
+ buf.putInt(drRec.cacheId());
+ buf.putLong(drRec.pageId());
+
+ buf.put((byte)drRec.itemId());
+
+ break;
+
+ case DATA_PAGE_SET_FREE_LIST_PAGE:
+ DataPageSetFreeListPageRecord freeListRec = (DataPageSetFreeListPageRecord)record;
+
+ buf.putInt(freeListRec.cacheId());
+ buf.putLong(freeListRec.pageId());
+
+ buf.putLong(freeListRec.freeListPage());
+
+ break;
+
+ case INIT_NEW_PAGE_RECORD:
+ InitNewPageRecord inpRec = (InitNewPageRecord)record;
+
+ buf.putInt(inpRec.cacheId());
+ buf.putLong(inpRec.pageId());
+
+ buf.putShort((short)inpRec.ioType());
+ buf.putShort((short)inpRec.ioVersion());
+ buf.putLong(inpRec.newPageId());
+
+ break;
+
+ case BTREE_META_PAGE_INIT_ROOT:
+ MetaPageInitRootRecord imRec = (MetaPageInitRootRecord)record;
+
+ buf.putInt(imRec.cacheId());
+ buf.putLong(imRec.pageId());
+
+ buf.putLong(imRec.rootId());
+
+ break;
+
+ case BTREE_META_PAGE_INIT_ROOT2:
+ MetaPageInitRootInlineRecord imRec2 = (MetaPageInitRootInlineRecord)record;
+
+ buf.putInt(imRec2.cacheId());
+ buf.putLong(imRec2.pageId());
+
+ buf.putLong(imRec2.rootId());
+
+ buf.putShort((short)imRec2.inlineSize());
+ break;
+
+ case BTREE_META_PAGE_ADD_ROOT:
+ MetaPageAddRootRecord arRec = (MetaPageAddRootRecord)record;
+
+ buf.putInt(arRec.cacheId());
+ buf.putLong(arRec.pageId());
+
+ buf.putLong(arRec.rootId());
+
+ break;
+
+ case BTREE_META_PAGE_CUT_ROOT:
+ MetaPageCutRootRecord crRec = (MetaPageCutRootRecord)record;
+
+ buf.putInt(crRec.cacheId());
+ buf.putLong(crRec.pageId());
+
+ break;
+
+ case BTREE_INIT_NEW_ROOT:
+ NewRootInitRecord<?> riRec = (NewRootInitRecord<?>)record;
+
+ buf.putInt(riRec.cacheId());
+ buf.putLong(riRec.pageId());
+
+ buf.putLong(riRec.rootId());
+ buf.putShort((short)riRec.io().getType());
+ buf.putShort((short)riRec.io().getVersion());
+ buf.putLong(riRec.leftId());
+ buf.putLong(riRec.rightId());
+
+ putRow(buf, riRec.rowBytes());
+
+ break;
+
+ case BTREE_PAGE_RECYCLE:
+ RecycleRecord recRec = (RecycleRecord)record;
+
+ buf.putInt(recRec.cacheId());
+ buf.putLong(recRec.pageId());
+
+ buf.putLong(recRec.newPageId());
+
+ break;
+
+ case BTREE_PAGE_INSERT:
+ InsertRecord<?> inRec = (InsertRecord<?>)record;
+
+ buf.putInt(inRec.cacheId());
+ buf.putLong(inRec.pageId());
+
+ buf.putShort((short)inRec.io().getType());
+ buf.putShort((short)inRec.io().getVersion());
+ buf.putShort((short)inRec.index());
+ buf.putLong(inRec.rightId());
+
+ putRow(buf, inRec.rowBytes());
+
+ break;
+
+ case BTREE_FIX_LEFTMOST_CHILD:
+ FixLeftmostChildRecord flRec = (FixLeftmostChildRecord)record;
+
+ buf.putInt(flRec.cacheId());
+ buf.putLong(flRec.pageId());
+
+ buf.putLong(flRec.rightId());
+
+ break;
+
+ case BTREE_FIX_COUNT:
+ FixCountRecord fcRec = (FixCountRecord)record;
+
+ buf.putInt(fcRec.cacheId());
+ buf.putLong(fcRec.pageId());
+
+ buf.putShort((short)fcRec.count());
+
+ break;
+
+ case BTREE_PAGE_REPLACE:
+ ReplaceRecord<?> rRec = (ReplaceRecord<?>)record;
+
+ buf.putInt(rRec.cacheId());
+ buf.putLong(rRec.pageId());
+
+ buf.putShort((short)rRec.io().getType());
+ buf.putShort((short)rRec.io().getVersion());
+ buf.putShort((short)rRec.index());
+
+ putRow(buf, rRec.rowBytes());
+
+ break;
+
+ case BTREE_PAGE_REMOVE:
+ RemoveRecord rmRec = (RemoveRecord)record;
+
+ buf.putInt(rmRec.cacheId());
+ buf.putLong(rmRec.pageId());
+
+ buf.putShort((short)rmRec.index());
+ buf.putShort((short)rmRec.count());
+
+ break;
+
+ case BTREE_PAGE_INNER_REPLACE:
+ InnerReplaceRecord<?> irRec = (InnerReplaceRecord<?>)record;
+
+ buf.putInt(irRec.cacheId());
+ buf.putLong(irRec.pageId());
+
+ buf.putShort((short)irRec.destinationIndex());
+ buf.putLong(irRec.sourcePageId());
+ buf.putShort((short)irRec.sourceIndex());
+ buf.putLong(irRec.removeId());
+
+ break;
+
+ case BTREE_FORWARD_PAGE_SPLIT:
+ SplitForwardPageRecord sfRec = (SplitForwardPageRecord)record;
+
+ buf.putInt(sfRec.cacheId());
+ buf.putLong(sfRec.pageId());
+
+ buf.putLong(sfRec.forwardId());
+ buf.putShort((short)sfRec.ioType());
+ buf.putShort((short)sfRec.ioVersion());
+ buf.putLong(sfRec.sourcePageId());
+ buf.putShort((short)sfRec.middleIndex());
+ buf.putShort((short)sfRec.count());
+
+ break;
+
+ case BTREE_EXISTING_PAGE_SPLIT:
+ SplitExistingPageRecord seRec = (SplitExistingPageRecord)record;
+
+ buf.putInt(seRec.cacheId());
+ buf.putLong(seRec.pageId());
+
+ buf.putShort((short)seRec.middleIndex());
+ buf.putLong(seRec.forwardId());
+
+ break;
+
+ case BTREE_PAGE_MERGE:
+ MergeRecord<?> mRec = (MergeRecord<?>)record;
+
+ buf.putInt(mRec.cacheId());
+ buf.putLong(mRec.pageId());
+
+ buf.putLong(mRec.parentId());
+ buf.putShort((short)mRec.parentIndex());
+ buf.putLong(mRec.rightId());
+ buf.put((byte)(mRec.isEmptyBranch() ? 1 : 0));
+
+ break;
+
+ case PAGES_LIST_SET_NEXT:
+ PagesListSetNextRecord plNextRec = (PagesListSetNextRecord)record;
+
+ buf.putInt(plNextRec.cacheId());
+ buf.putLong(plNextRec.pageId());
+
+ buf.putLong(plNextRec.nextPageId());
+
+ break;
+
+ case PAGES_LIST_SET_PREVIOUS:
+ PagesListSetPreviousRecord plPrevRec = (PagesListSetPreviousRecord)record;
+
+ buf.putInt(plPrevRec.cacheId());
+ buf.putLong(plPrevRec.pageId());
+
+ buf.putLong(plPrevRec.previousPageId());
+
+ break;
+
+ case PAGES_LIST_INIT_NEW_PAGE:
+ PagesListInitNewPageRecord plNewRec = (PagesListInitNewPageRecord)record;
+
+ buf.putInt(plNewRec.cacheId());
+ buf.putLong(plNewRec.pageId());
+ buf.putInt(plNewRec.ioType());
+ buf.putInt(plNewRec.ioVersion());
+ buf.putLong(plNewRec.newPageId());
+
+ buf.putLong(plNewRec.previousPageId());
+ buf.putLong(plNewRec.dataPageId());
+
+ break;
+
+ case PAGES_LIST_ADD_PAGE:
+ PagesListAddPageRecord plAddRec = (PagesListAddPageRecord)record;
+
+ buf.putInt(plAddRec.cacheId());
+ buf.putLong(plAddRec.pageId());
+
+ buf.putLong(plAddRec.dataPageId());
+
+ break;
+
+ case PAGES_LIST_REMOVE_PAGE:
+ PagesListRemovePageRecord plRmvRec = (PagesListRemovePageRecord)record;
+
+ buf.putInt(plRmvRec.cacheId());
+ buf.putLong(plRmvRec.pageId());
+
+ buf.putLong(plRmvRec.removedPageId());
+
+ break;
+
+ case BTREE_FIX_REMOVE_ID:
+ FixRemoveId frRec = (FixRemoveId)record;
+
+ buf.putInt(frRec.cacheId());
+ buf.putLong(frRec.pageId());
+
+ buf.putLong(frRec.removeId());
+
+ break;
+
+ case TRACKING_PAGE_DELTA:
+ TrackingPageDeltaRecord tpDelta = (TrackingPageDeltaRecord)record;
+
+ buf.putInt(tpDelta.cacheId());
+ buf.putLong(tpDelta.pageId());
+
+ buf.putLong(tpDelta.pageIdToMark());
+ buf.putLong(tpDelta.nextSnapshotId());
+ buf.putLong(tpDelta.lastSuccessfulSnapshotId());
+
+ break;
+
+ case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
+ MetaPageUpdateNextSnapshotId mpUpdateNextSnapshotId = (MetaPageUpdateNextSnapshotId)record;
+
+ buf.putInt(mpUpdateNextSnapshotId.cacheId());
+ buf.putLong(mpUpdateNextSnapshotId.pageId());
+
+ buf.putLong(mpUpdateNextSnapshotId.nextSnapshotId());
+
+ break;
+
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
+ MetaPageUpdateLastSuccessfulFullSnapshotId mpUpdateLastSuccFullSnapshotId =
+ (MetaPageUpdateLastSuccessfulFullSnapshotId)record;
+
+ buf.putInt(mpUpdateLastSuccFullSnapshotId.cacheId());
+ buf.putLong(mpUpdateLastSuccFullSnapshotId.pageId());
+
+ buf.putLong(mpUpdateLastSuccFullSnapshotId.lastSuccessfulFullSnapshotId());
+
+ break;
+
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
+ MetaPageUpdateLastSuccessfulSnapshotId mpUpdateLastSuccSnapshotId =
+ (MetaPageUpdateLastSuccessfulSnapshotId)record;
+
+ buf.putInt(mpUpdateLastSuccSnapshotId.cacheId());
+ buf.putLong(mpUpdateLastSuccSnapshotId.pageId());
+
+ buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotId());
+ buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotTag());
+
+ break;
+
+ case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
+ MetaPageUpdateLastAllocatedIndex mpUpdateLastAllocatedIdx =
+ (MetaPageUpdateLastAllocatedIndex) record;
+
+ buf.putInt(mpUpdateLastAllocatedIdx.cacheId());
+ buf.putLong(mpUpdateLastAllocatedIdx.pageId());
+
+ buf.putInt(mpUpdateLastAllocatedIdx.lastAllocatedIndex());
+
+ break;
+
+ case PART_META_UPDATE_STATE:
+ PartitionMetaStateRecord partMetaStateRecord = (PartitionMetaStateRecord) record;
+
+ buf.putInt(partMetaStateRecord.cacheId());
+
+ buf.putInt(partMetaStateRecord.partitionId());
+
+ buf.put(partMetaStateRecord.state());
+
+ buf.putLong(partMetaStateRecord.updateCounter());
+
+ break;
+
+ case PAGE_LIST_META_RESET_COUNT_RECORD:
+ PageListMetaResetCountRecord pageListMetaResetCntRecord = (PageListMetaResetCountRecord) record;
+
+ buf.putInt(pageListMetaResetCntRecord.cacheId());
+ buf.putLong(pageListMetaResetCntRecord.pageId());
+
+ break;
+
+ default:
+ throw new UnsupportedOperationException("Type: " + record.type());
+ }
+
+ if (!skipCrc) {
+ int curPos = buf.position();
+
+ buf.position(startPos);
+
+ // This call will move buffer position to the end of the record again.
+ int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos);
+
+ buf.putInt(crcVal);
+ }
+ else
+ buf.putInt(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALRecord readRecord(FileInput in0, WALPointer expPtr) throws IOException, IgniteCheckedException {
+ long startPos = -1;
+
+ try (FileInput.Crc32CheckingFileInput in = in0.startRead(skipCrc)) {
+ startPos = in0.position();
+
+ WALRecord res = readRecord(in, expPtr);
+
+ assert res != null;
+
+ res.size((int)(in0.position() - startPos + 4)); // Account for CRC which will be read afterwards.
+
+ return res;
+ }
+ catch (EOFException | SegmentEofException e) {
+ throw e;
+ }
+ catch (Exception e) {
+ throw new IgniteCheckedException("Failed to read WAL record at position: " + startPos, e);
+ }
+ }
+
+ /**
+ * @param in In.
+ */
+ private WALRecord readRecord(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException {
+ int type = in.readUnsignedByte();
+
+ if (type == 0)
+ throw new SegmentEofException("Reached logical end of the segment", null);
+
+ FileWALPointer ptr = readPosition(in);
+
+ if (!F.eq(ptr, expPtr))
+ throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr +
+ ", readPtr=" + ptr + ']', null);
+
+ RecordType recType = RecordType.fromOrdinal(type - 1);
+
+ if (recType == null)
+ throw new IOException("Unknown record type: " + type);
+
+ WALRecord res;
+
+ switch (recType) {
+ case PAGE_RECORD:
+ byte[] arr = new byte[pageSize];
+
+ int cacheId = in.readInt();
+ long pageId = in.readLong();
+
+ in.readFully(arr);
+
+ res = new PageSnapshot(new FullPageId(pageId, cacheId), arr);
+
+ break;
+
+ case CHECKPOINT_RECORD:
+ long msb = in.readLong();
+ long lsb = in.readLong();
+ boolean hasPtr = in.readByte() != 0;
+ int idx = hasPtr ? in.readInt() : 0;
+ int offset = hasPtr ? in.readInt() : 0;
+ int len = hasPtr ? in.readInt() : 0;
+
+ Map<Integer, CacheState> states = readPartitionStates(in);
+
+ boolean end = in.readByte() != 0;
+
+ FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx, offset, len) : null;
+
+ CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end);
+
+ cpRec.cacheGroupStates(states);
+
+ res = cpRec;
+
+ break;
+
+ case META_PAGE_INIT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int ioType = in.readUnsignedShort();
+ int ioVer = in.readUnsignedShort();
+ long treeRoot = in.readLong();
+ long reuseListRoot = in.readLong();
+
+ res = new MetaPageInitRecord(cacheId, pageId, ioType, ioVer, treeRoot, reuseListRoot);
+
+ break;
+
+ case PARTITION_META_PAGE_UPDATE_COUNTERS:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long updCntr = in.readLong();
+ long rmvId = in.readLong();
+ int partSize = in.readInt();
+ long countersPageId = in.readLong();
+ byte state = in.readByte();
+ int allocatedIdxCandidate = in.readInt();
+
+ res = new MetaPageUpdatePartitionDataRecord(cacheId, pageId, updCntr, rmvId, partSize, countersPageId, state, allocatedIdxCandidate);
+
+ break;
+
+ case MEMORY_RECOVERY:
+ long ts = in.readLong();
+
+ res = new MemoryRecoveryRecord(ts);
+
+ break;
+
+ case PARTITION_DESTROY:
+ cacheId = in.readInt();
+ int partId = in.readInt();
+
+ res = new PartitionDestroyRecord(cacheId, partId);
+
+ break;
+
+ case DATA_RECORD:
+ int entryCnt = in.readInt();
+
+ List<DataEntry> entries = new ArrayList<>(entryCnt);
+
+ for (int i = 0; i < entryCnt; i++)
+ entries.add(readDataEntry(in));
+
+ res = new DataRecord(entries);
+
+ break;
+
+ case HEADER_RECORD:
+ long magic = in.readLong();
+
+ if (magic != HeaderRecord.MAGIC)
+ throw new EOFException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) +
+ ", actual=" + U.hexLong(magic) + ']');
+
+ int ver = in.readInt();
+
+ res = new HeaderRecord(ver);
+
+ break;
+
+ case DATA_PAGE_INSERT_RECORD: {
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int size = in.readUnsignedShort();
+
+ in.ensure(size);
+
+ byte[] payload = new byte[size];
+
+ in.readFully(payload);
+
+ res = new DataPageInsertRecord(cacheId, pageId, payload);
+
+ break;
+ }
+
+ case DATA_PAGE_UPDATE_RECORD: {
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int itemId = in.readInt();
+
+ int size = in.readUnsignedShort();
+
+ in.ensure(size);
+
+ byte[] payload = new byte[size];
+
+ in.readFully(payload);
+
+ res = new DataPageUpdateRecord(cacheId, pageId, itemId, payload);
+
+ break;
+ }
+
+ case DATA_PAGE_INSERT_FRAGMENT_RECORD: {
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ final long lastLink = in.readLong();
+ final int payloadSize = in.readInt();
+
+ final byte[] payload = new byte[payloadSize];
+
+ in.readFully(payload);
+
+ res = new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink);
+
+ break;
+ }
+
+ case DATA_PAGE_REMOVE_RECORD:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int itemId = in.readUnsignedByte();
+
+ res = new DataPageRemoveRecord(cacheId, pageId, itemId);
+
+ break;
+
+ case DATA_PAGE_SET_FREE_LIST_PAGE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long freeListPage = in.readLong();
+
+ res = new DataPageSetFreeListPageRecord(cacheId, pageId, freeListPage);
+
+ break;
+
+ case INIT_NEW_PAGE_RECORD:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ ioType = in.readUnsignedShort();
+ ioVer = in.readUnsignedShort();
+ long virtualPageId = in.readLong();
+
+ res = new InitNewPageRecord(cacheId, pageId, ioType, ioVer, virtualPageId);
+
+ break;
+
+ case BTREE_META_PAGE_INIT_ROOT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long rootId = in.readLong();
+
+ res = new MetaPageInitRootRecord(cacheId, pageId, rootId);
+
+ break;
+
+ case BTREE_META_PAGE_INIT_ROOT2:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long rootId2 = in.readLong();
+ int inlineSize = in.readShort();
+
+ res = new MetaPageInitRootInlineRecord(cacheId, pageId, rootId2, inlineSize);
+
+ break;
+
+ case BTREE_META_PAGE_ADD_ROOT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ rootId = in.readLong();
+
+ res = new MetaPageAddRootRecord(cacheId, pageId, rootId);
+
+ break;
+
+ case BTREE_META_PAGE_CUT_ROOT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ res = new MetaPageCutRootRecord(cacheId, pageId);
+
+ break;
+
+ case BTREE_INIT_NEW_ROOT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ rootId = in.readLong();
+ ioType = in.readUnsignedShort();
+ ioVer = in.readUnsignedShort();
+ long leftId = in.readLong();
+ long rightId = in.readLong();
+
+ BPlusIO<?> io = BPlusIO.getBPlusIO(ioType, ioVer);
+
+ byte[] rowBytes = new byte[io.getItemSize()];
+
+ in.readFully(rowBytes);
+
+ res = new NewRootInitRecord<>(cacheId, pageId, rootId, (BPlusInnerIO<?>)io, leftId, rowBytes, rightId);
+
+ break;
+
+ case BTREE_PAGE_RECYCLE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long newPageId = in.readLong();
+
+ res = new RecycleRecord(cacheId, pageId, newPageId);
+
+ break;
+
+ case BTREE_PAGE_INSERT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ ioType = in.readUnsignedShort();
+ ioVer = in.readUnsignedShort();
+ int itemIdx = in.readUnsignedShort();
+ rightId = in.readLong();
+
+ io = BPlusIO.getBPlusIO(ioType, ioVer);
+
+ rowBytes = new byte[io.getItemSize()];
+
+ in.readFully(rowBytes);
+
+ res = new InsertRecord<>(cacheId, pageId, io, itemIdx, rowBytes, rightId);
+
+ break;
+
+ case BTREE_FIX_LEFTMOST_CHILD:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ rightId = in.readLong();
+
+ res = new FixLeftmostChildRecord(cacheId, pageId, rightId);
+
+ break;
+
+ case BTREE_FIX_COUNT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int cnt = in.readUnsignedShort();
+
+ res = new FixCountRecord(cacheId, pageId, cnt);
+
+ break;
+
+ case BTREE_PAGE_REPLACE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ ioType = in.readUnsignedShort();
+ ioVer = in.readUnsignedShort();
+ itemIdx = in.readUnsignedShort();
+
+ io = BPlusIO.getBPlusIO(ioType, ioVer);
+
+ rowBytes = new byte[io.getItemSize()];
+
+ in.readFully(rowBytes);
+
+ res = new ReplaceRecord<>(cacheId, pageId, io, rowBytes, itemIdx);
+
+ break;
+
+ case BTREE_PAGE_REMOVE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ itemIdx = in.readUnsignedShort();
+ cnt = in.readUnsignedShort();
+
+ res = new RemoveRecord(cacheId, pageId, itemIdx, cnt);
+
+ break;
+
+ case BTREE_PAGE_INNER_REPLACE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int dstIdx = in.readUnsignedShort();
+ long srcPageId = in.readLong();
+ int srcIdx = in.readUnsignedShort();
+ rmvId = in.readLong();
+
+ res = new InnerReplaceRecord<>(cacheId, pageId, dstIdx, srcPageId, srcIdx, rmvId);
+
+ break;
+
+ case BTREE_FORWARD_PAGE_SPLIT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long fwdId = in.readLong();
+ ioType = in.readUnsignedShort();
+ ioVer = in.readUnsignedShort();
+ srcPageId = in.readLong();
+ int mid = in.readUnsignedShort();
+ cnt = in.readUnsignedShort();
+
+ res = new SplitForwardPageRecord(cacheId, pageId, fwdId, ioType, ioVer, srcPageId, mid, cnt);
+
+ break;
+
+ case BTREE_EXISTING_PAGE_SPLIT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ mid = in.readUnsignedShort();
+ fwdId = in.readLong();
+
+ res = new SplitExistingPageRecord(cacheId, pageId, mid, fwdId);
+
+ break;
+
+ case BTREE_PAGE_MERGE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long prntId = in.readLong();
+ int prntIdx = in.readUnsignedShort();
+ rightId = in.readLong();
+ boolean emptyBranch = in.readBoolean();
+
+ res = new MergeRecord<>(cacheId, pageId, prntId, prntIdx, rightId, emptyBranch);
+
+ break;
+
+ case BTREE_FIX_REMOVE_ID:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ rmvId = in.readLong();
+
+ res = new FixRemoveId(cacheId, pageId, rmvId);
+
+ break;
+
+ case PAGES_LIST_SET_NEXT:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+ long nextPageId = in.readLong();
+
+ res = new PagesListSetNextRecord(cacheId, pageId, nextPageId);
+
+ break;
+
+ case PAGES_LIST_SET_PREVIOUS:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+ long prevPageId = in.readLong();
+
+ res = new PagesListSetPreviousRecord(cacheId, pageId, prevPageId);
+
+ break;
+
+ case PAGES_LIST_INIT_NEW_PAGE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+ ioType = in.readInt();
+ ioVer = in.readInt();
+ newPageId = in.readLong();
+ prevPageId = in.readLong();
+ long addDataPageId = in.readLong();
+
+ res = new PagesListInitNewPageRecord(cacheId, pageId, ioType, ioVer, newPageId, prevPageId, addDataPageId);
+
+ break;
+
+ case PAGES_LIST_ADD_PAGE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+ long dataPageId = in.readLong();
+
+ res = new PagesListAddPageRecord(cacheId, pageId, dataPageId);
+
+ break;
+
+ case PAGES_LIST_REMOVE_PAGE:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+ long rmvdPageId = in.readLong();
+
+ res = new PagesListRemovePageRecord(cacheId, pageId, rmvdPageId);
+
+ break;
+
+ case TRACKING_PAGE_DELTA:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long pageIdToMark = in.readLong();
+ long nextSnapshotId0 = in.readLong();
+ long lastSuccessfulSnapshotId0 = in.readLong();
+
+ res = new TrackingPageDeltaRecord(cacheId, pageId, pageIdToMark, nextSnapshotId0, lastSuccessfulSnapshotId0);
+
+ break;
+
+ case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long nextSnapshotId = in.readLong();
+
+ res = new MetaPageUpdateNextSnapshotId(cacheId, pageId, nextSnapshotId);
+
+ break;
+
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long lastSuccessfulFullSnapshotId = in.readLong();
+
+ res = new MetaPageUpdateLastSuccessfulFullSnapshotId(cacheId, pageId, lastSuccessfulFullSnapshotId);
+
+ break;
+
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ long lastSuccessfulSnapshotId = in.readLong();
+ long lastSuccessfulSnapshotTag = in.readLong();
+
+ res = new MetaPageUpdateLastSuccessfulSnapshotId(cacheId, pageId, lastSuccessfulSnapshotId, lastSuccessfulSnapshotTag);
+
+ break;
+
+ case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ int lastAllocatedIdx = in.readInt();
+
+ res = new MetaPageUpdateLastAllocatedIndex(cacheId, pageId, lastAllocatedIdx);
+
+ break;
+
+ case PART_META_UPDATE_STATE:
+ cacheId = in.readInt();
+ partId = in.readInt();
+
+ state = in.readByte();
+
+ long updateCounter = in.readLong();
+
+ res = new PartitionMetaStateRecord(cacheId, partId, GridDhtPartitionState.fromOrdinal(state), updateCounter);
+
+ break;
+
+ case PAGE_LIST_META_RESET_COUNT_RECORD:
+ cacheId = in.readInt();
+ pageId = in.readLong();
+
+ res = new PageListMetaResetCountRecord(cacheId, pageId);
+ break;
+
+ case SWITCH_SEGMENT_RECORD:
+ throw new EOFException("END OF SEGMENT");
+
+ default:
+ throw new UnsupportedOperationException("Type: " + recType);
+ }
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("CastConflictsWithInstanceof")
+ @Override public int size(WALRecord record) throws IgniteCheckedException {
+ int commonFields = /* Type */1 + /* Pointer */12 + /*CRC*/4;
+
+ switch (record.type()) {
+ case PAGE_RECORD:
+ assert record instanceof PageSnapshot;
+
+ PageSnapshot pageRec = (PageSnapshot)record;
+
+ return commonFields + pageRec.pageData().length + 12;
+
+ case CHECKPOINT_RECORD:
+ CheckpointRecord cpRec = (CheckpointRecord)record;
+
+ assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
+ "Invalid WAL record: " + cpRec;
+
+ int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates());
+
+ FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
+
+ return commonFields + 18 + cacheStatesSize + (walPtr == null ? 0 : 16);
+
+ case META_PAGE_INIT:
+ return commonFields + /*cache ID*/4 + /*page ID*/8 + /*ioType*/2 + /*ioVer*/2 + /*tree root*/8 + /*reuse root*/8;
+
+ case PARTITION_META_PAGE_UPDATE_COUNTERS:
+ return commonFields + /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1
+ + /*allocatedIdxCandidate*/ 4;
+
+ case MEMORY_RECOVERY:
+ return commonFields + 8;
+
+ case PARTITION_DESTROY:
+ return commonFields + /*cacheId*/4 + /*partId*/4;
+
+ case DATA_RECORD:
+ DataRecord dataRec = (DataRecord)record;
+
+ return commonFields + 4 + dataSize(dataRec);
+
+ case HEADER_RECORD:
+ return HEADER_RECORD_SIZE;
+
+ case DATA_PAGE_INSERT_RECORD:
+ DataPageInsertRecord diRec = (DataPageInsertRecord)record;
+
+ return commonFields + 4 + 8 + 2 + diRec.payload().length;
+
+ case DATA_PAGE_UPDATE_RECORD:
+ DataPageUpdateRecord uRec = (DataPageUpdateRecord)record;
+
+ return commonFields + 4 + 8 + 2 + 4 +
+ uRec.payload().length;
+
+ case DATA_PAGE_INSERT_FRAGMENT_RECORD:
+ final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record;
+
+ return commonFields + 4 + 8 + 8 + 4 + difRec.payloadSize();
+
+ case DATA_PAGE_REMOVE_RECORD:
+ return commonFields + 4 + 8 + 1;
+
+ case DATA_PAGE_SET_FREE_LIST_PAGE:
+ return commonFields + 4 + 8 + 8;
+
+ case INIT_NEW_PAGE_RECORD:
+ return commonFields + 4 + 8 + 2 + 2 + 8;
+
+ case BTREE_META_PAGE_INIT_ROOT:
+ return commonFields + 4 + 8 + 8;
+
+ case BTREE_META_PAGE_INIT_ROOT2:
+ return commonFields + 4 + 8 + 8 + 2;
+
+ case BTREE_META_PAGE_ADD_ROOT:
+ return commonFields + 4 + 8 + 8;
+
+ case BTREE_META_PAGE_CUT_ROOT:
+ return commonFields + 4 + 8;
+
+ case BTREE_INIT_NEW_ROOT:
+ NewRootInitRecord<?> riRec = (NewRootInitRecord<?>)record;
+
+ return commonFields + 4 + 8 + 8 + 2 + 2 + 8 + 8 + riRec.io().getItemSize();
+
+ case BTREE_PAGE_RECYCLE:
+ return commonFields + 4 + 8 + 8;
+
+ case BTREE_PAGE_INSERT:
+ InsertRecord<?> inRec = (InsertRecord<?>)record;
+
+ return commonFields + 4 + 8 + 2 + 2 + 2 + 8 + inRec.io().getItemSize();
+
+ case BTREE_FIX_LEFTMOST_CHILD:
+ return commonFields + 4 + 8 + 8;
+
+ case BTREE_FIX_COUNT:
+ return commonFields + 4 + 8 + 2;
+
+ case BTREE_PAGE_REPLACE:
+ ReplaceRecord<?> rRec = (ReplaceRecord<?>)record;
+
+ return commonFields + 4 + 8 + 2 + 2 + 2 + rRec.io().getItemSize();
+
+ case BTREE_PAGE_REMOVE:
+ return commonFields + 4 + 8 + 2 + 2;
+
+ case BTREE_PAGE_INNER_REPLACE:
+ return commonFields + 4 + 8 + 2 + 8 + 2 + 8;
+
+ case BTREE_FORWARD_PAGE_SPLIT:
+ return commonFields + 4 + 8 + 8 + 2 + 2 + 8 + 2 + 2;
+
+ case BTREE_EXISTING_PAGE_SPLIT:
+ return commonFields + 4 + 8 + 2 + 8;
+
+ case BTREE_PAGE_MERGE:
+ return commonFields + 4 + 8 + 8 + 2 + 8 + 1;
+
+ case BTREE_FIX_REMOVE_ID:
+ return commonFields + 4 + 8 + 8;
+
+ case PAGES_LIST_SET_NEXT:
+ return commonFields + 4 + 8 + 8;
+
+ case PAGES_LIST_SET_PREVIOUS:
+ return commonFields + 4 + 8 + 8;
+
+ case PAGES_LIST_INIT_NEW_PAGE:
+ return commonFields + 4 + 8 + 4 + 4 + 8 + 8 + 8;
+
+ case PAGES_LIST_ADD_PAGE:
+ return commonFields + 4 + 8 + 8;
+
+ case PAGES_LIST_REMOVE_PAGE:
+ return commonFields + 4 + 8 + 8;
+
+ case TRACKING_PAGE_DELTA:
+ return commonFields + 4 + 8 + 8 + 8 + 8;
+
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
+ return commonFields + 4 + 8 + 8 + 8;
+
+ case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
+ return commonFields + 4 + 8 + 8;
+
+ case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
+ return commonFields + 4 + 8 + 8;
+
+ case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
+ return commonFields + 4 + 8 + 4;
+
+ case PART_META_UPDATE_STATE:
+ return commonFields + /*cacheId*/ 4 + /*partId*/ 4 + /*State*/1 + /*Update Counter*/ 8;
+
+ case PAGE_LIST_META_RESET_COUNT_RECORD:
+ return commonFields + /*cacheId*/ 4 + /*pageId*/ 8;
+
+ case SWITCH_SEGMENT_RECORD:
+ return commonFields;
+
+ default:
+ throw new UnsupportedOperationException("Type: " + record.type());
+ }
+ }
+
+ /**
+ * @param buf Byte buffer to serialize version to.
+ * @param ptr File WAL pointer to write.
+ */
+ private void putPosition(ByteBuffer buf, FileWALPointer ptr) {
+ buf.putLong(ptr.index());
+ buf.putInt(ptr.fileOffset());
+ }
+
+ /**
+ * @param in Data input to read pointer from.
+ * @return Read file WAL pointer.
+ * @throws IOException If failed to write.
+ */
+ private FileWALPointer readPosition(DataInput in) throws IOException {
+ long idx = in.readLong();
+ int fileOffset = in.readInt();
+
+ return new FileWALPointer(idx, fileOffset, 0);
+ }
+
+ /**
+ * @param dataRec Data record to serialize.
+ * @return Full data record size.
+ * @throws IgniteCheckedException If failed to obtain the length of one of the entries.
+ */
+ private int dataSize(DataRecord dataRec) throws IgniteCheckedException {
+ int sz = 0;
+
+ for (DataEntry entry : dataRec.writeEntries())
+ sz += entrySize(entry);
+
+ return sz;
+ }
+
+ /**
+ * @param entry Entry to get size for.
+ * @return Entry size.
+ * @throws IgniteCheckedException If failed to get key or value bytes length.
+ */
+ private int entrySize(DataEntry entry) throws IgniteCheckedException {
+ GridCacheContext cctx = this.cctx.cacheContext(entry.cacheId());
+ CacheObjectContext coCtx = cctx.cacheObjectContext();
+
+ return
+ /*cache ID*/4 +
+ /*key*/entry.key().valueBytesLength(coCtx) +
+ /*value*/(entry.value() == null ? 4 : entry.value().valueBytesLength(coCtx)) +
+ /*op*/1 +
+ /*near xid ver*/CacheVersionIO.size(entry.nearXidVersion(), true) +
+ /*write ver*/CacheVersionIO.size(entry.writeVersion(), false) +
+ /*part ID*/4 +
+ /*expire Time*/8 +
+ /*part cnt*/8;
+ }
+
+ /**
+ * @param states Partition states.
+ * @return Size required to write partition states.
+ */
+ private int cacheStatesSize(Map<Integer, CacheState> states) {
+ // Need 4 bytes for the number of caches.
+ int size = 2;
+
+ for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
+ // Cache ID.
+ size += 4;
+
+ // Need 2 bytes for the number of partitions.
+ size += 2;
+
+ CacheState state = entry.getValue();
+
+ // 2 bytes partition ID, size and counter per partition.
+ size += 18 * state.size();
+ }
+
+ return size;
+ }
+
+ /**
+ * @param buf Buffer to write to.
+ * @param entry Data entry.
+ */
+ private void putDataEntry(ByteBuffer buf, DataEntry entry) throws IgniteCheckedException {
+ buf.putInt(entry.cacheId());
+
+ if (!entry.key().putValue(buf))
+ throw new AssertionError();
+
+ if (entry.value() == null)
+ buf.putInt(-1);
+ else if (!entry.value().putValue(buf))
+ throw new AssertionError();
+
+ buf.put((byte)entry.op().ordinal());
+
+ putVersion(buf, entry.nearXidVersion(), true);
+ putVersion(buf, entry.writeVersion(), false);
+
+ buf.putInt(entry.partitionId());
+ buf.putLong(entry.partitionCounter());
+ buf.putLong(entry.expireTime());
+ }
+
+ /**
+ * @param states Cache states.
+ */
+ private void putCacheStates(ByteBuffer buf, Map<Integer, CacheState> states) {
+ buf.putShort((short)states.size());
+
+ for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
+ buf.putInt(entry.getKey());
+
+ CacheState state = entry.getValue();
+
+ // Need 2 bytes for the number of partitions.
+ buf.putShort((short)state.size());
+
+ for (int i = 0; i < state.size(); i++) {
+ buf.putShort((short)state.partitionByIndex(i));
+
+ buf.putLong(state.partitionSizeByIndex(i));
+ buf.putLong(state.partitionCounterByIndex(i));
+ }
+ }
+ }
+
+ /**
+ * @param in Input to read from.
+ * @return Read entry.
+ */
+ private DataEntry readDataEntry(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
+ int cacheId = in.readInt();
+
+ int keySize = in.readInt();
+ byte keyType = in.readByte();
+ byte[] keyBytes = new byte[keySize];
+ in.readFully(keyBytes);
+
+ int valSize = in.readInt();
+
+ byte valType = 0;
+ byte[] valBytes = null;
+
+ if (valSize >= 0) {
+ valType = in.readByte();
+ valBytes = new byte[valSize];
+ in.readFully(valBytes);
+ }
+
+ byte ord = in.readByte();
+
+ GridCacheOperation op = GridCacheOperation.fromOrdinal(ord & 0xFF);
+
+ GridCacheVersion nearXidVer = readVersion(in, true);
+ GridCacheVersion writeVer = readVersion(in, false);
+
+ int partId = in.readInt();
+ long partCntr = in.readLong();
+ long expireTime = in.readLong();
+
+ GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+
+ if (cacheCtx != null) {
+ CacheObjectContext coCtx = cacheCtx.cacheObjectContext();
+
+ KeyCacheObject key = co.toKeyCacheObject(coCtx, keyType, keyBytes);
+ CacheObject val = valBytes != null ? co.toCacheObject(coCtx, valType, valBytes) : null;
+
+ return new DataEntry(
+ cacheId,
+ key,
+ val,
+ op,
+ nearXidVer,
+ writeVer,
+ expireTime,
+ partId,
+ partCntr
+ );
+ }
+ else
+ return new LazyDataEntry(
+ cctx,
+ cacheId,
+ keyType,
+ keyBytes,
+ valType,
+ valBytes,
+ op,
+ nearXidVer,
+ writeVer,
+ expireTime,
+ partId,
+ partCntr);
+ }
+
+ /**
+ * @param buf Buffer to read from.
+ * @return Read map.
+ */
+ private Map<Integer, CacheState> readPartitionStates(DataInput buf) throws IOException {
+ int caches = buf.readShort() & 0xFFFF;
+
+ if (caches == 0)
+ return Collections.emptyMap();
+
+ Map<Integer, CacheState> states = new HashMap<>(caches, 1.0f);
+
+ for (int i = 0; i < caches; i++) {
+ int cacheId = buf.readInt();
+
+ int parts = buf.readShort() & 0xFFFF;
+
+ CacheState state = new CacheState(parts);
+
+ for (int p = 0; p < parts; p++) {
+ int partId = buf.readShort() & 0xFFFF;
+ long size = buf.readLong();
+ long partCntr = buf.readLong();
+
+ state.addPartitionState(partId, size, partCntr);
+ }
+
+ states.put(cacheId, state);
+ }
+
+ return states;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param ver Version to write.
+ * @param allowNull Is {@code null}version allowed.
+ */
+ private void putVersion(ByteBuffer buf, GridCacheVersion ver, boolean allowNull) {
+ CacheVersionIO.write(buf, ver, allowNull);
+ }
+
+ /**
+ * Changes the buffer position by the number of read bytes.
+ *
+ * @param in Data input to read from.
+ * @param allowNull Is {@code null}version allowed.
+ * @return Read cache version.
+ */
+ private GridCacheVersion readVersion(ByteBufferBackedDataInput in, boolean allowNull) throws IOException {
+ // To be able to read serialization protocol version.
+ in.ensure(1);
+
+ try {
+ int size = CacheVersionIO.readSize(in.buffer(), allowNull);
+
+ in.ensure(size);
+
+ return CacheVersionIO.read(in.buffer(), allowNull);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param rowBytes Row bytes.
+ */
+ @SuppressWarnings("unchecked")
+ private static void putRow(ByteBuffer buf, byte[] rowBytes) {
+ assert rowBytes.length > 0;
+
+ buf.put(rowBytes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 0c264db..352a462 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -71,7 +71,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheManagerAdapter;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtUnreservedPartitionException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
index f264056..628111b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java
@@ -57,7 +57,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheManagerAdapter;
import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
import org.apache.ignite.internal.processors.continuous.GridContinuousHandler;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
index 643480d..e0751f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java
@@ -109,7 +109,11 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
@GridToStringExclude
private GridCacheSharedContext<?, ?> sharedCtx;
- //todo may be add init latch
+ /** */
+ private final ConcurrentHashMap<String, CacheInfo> cacheData = new ConcurrentHashMap<>();
+
+ /** */
+ private volatile CacheJoinNodeDiscoveryData localCacheData;
/** Listener. */
private final GridLocalEventListener lsr = new GridLocalEventListener() {
@@ -123,8 +127,8 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
final GridChangeGlobalStateFuture f = cgsLocFut.get();
if (f != null)
- f.initFut.listen(new CI1<IgniteInternalFuture>() {
- @Override public void apply(IgniteInternalFuture fut) {
+ f.initFut.listen(new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> fut) {
f.onDiscoveryEvent(e);
}
});
@@ -192,6 +196,9 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
ctx.event().addLocalEventListener(lsr, EVT_NODE_LEFT, EVT_NODE_FAILED);
}
+ /**
+ * @param data Joining node discovery data.
+ */
public void cacheProcessorStarted(CacheJoinNodeDiscoveryData data) {
assert data != null;
@@ -228,6 +235,15 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
}
/** {@inheritDoc} */
+ @Override public void onKernalStart() throws IgniteCheckedException {
+ super.onKernalStart();
+
+ // First node started (coordinator).
+ if (ctx.discovery().serverNodes(AffinityTopologyVersion.NONE).get(0).isLocal())
+ cacheData.putAll(localCacheData.caches());
+ }
+
+ /** {@inheritDoc} */
@Nullable @Override public DiscoveryDataExchangeType discoveryDataType() {
return DiscoveryDataExchangeType.STATE_PROC;
}
@@ -270,9 +286,10 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
*/
public IgniteInternalFuture<?> changeGlobalState(final boolean activate) {
if (cacheProc.transactions().tx() != null || sharedCtx.lockedTopologyVersion(null) != null)
- throw new IgniteException("Cannot " + prettyStr(activate) + " cluster, because cache locked on transaction.");
+ throw new IgniteException("Failed to " + prettyStr(activate) + " cluster (must invoke the " +
+ "method outside of an active transaction).");
- if ((globalState == ACTIVE && activate) || (this.globalState == INACTIVE && !activate))
+ if ((globalState == ACTIVE && activate) || (globalState == INACTIVE && !activate))
return new GridFinishedFuture<>();
final UUID requestId = UUID.randomUUID();
@@ -284,9 +301,10 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
if (locF.activate == activate)
return locF;
- else
- return new GridFinishedFuture<>(new IgniteException(
- "fail " + prettyStr(activate) + ", because now in progress" + prettyStr(locF.activate)));
+
+ return new GridFinishedFuture<>(new IgniteException(
+ "Failed to " + prettyStr(activate) + ", because another state change operation is currently " +
+ "in progress: " + prettyStr(locF.activate)));
}
if (ctx.clientNode()) {
@@ -295,7 +313,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
IgniteCompute comp = ((ClusterGroupAdapter)ctx.cluster().get().forServers()).compute();
if (log.isInfoEnabled())
- log.info("Send " + prettyStr(activate) + " request from client node [id=" +
+ log.info("Sending " + prettyStr(activate) + " request from client node [id=" +
ctx.localNodeId() + " topVer=" + topVer + " ]");
IgniteFuture<Void> fut = comp.runAsync(new ClientChangeGlobalStateComputeRequest(activate));
@@ -339,7 +357,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
"node is stopping."));
}
catch (IgniteCheckedException e) {
- log.error("Fail create or send change global state request." + cgsFut, e);
+ U.error(log, "Failed to create or send global state change request: " + cgsFut, e);
cgsFut.onDone(e);
}
@@ -352,6 +370,10 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
return cgsFut;
}
+ /**
+ * @param reqs Requests to print.
+ * @param active Active flag.
+ */
private void printCacheInfo(List<DynamicCacheChangeRequest> reqs, boolean active) {
assert reqs != null;
@@ -374,6 +396,9 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
log.info(sb.toString());
}
+ /**
+ * @param req Cache being started.
+ */
public void onCacheStart(DynamicCacheChangeRequest req) {
CacheInfo cacheInfo = cacheData.get(req.cacheName());
@@ -386,6 +411,9 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
);
}
+ /**
+ * @param req Cache being stopped.
+ */
public void onCacheStop(DynamicCacheChangeRequest req) {
CacheInfo cacheInfo = cacheData.get(req.cacheName());
@@ -393,6 +421,9 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
cacheData.remove(req.cacheName());
}
+ /**
+ * @return All caches map.
+ */
private Map<String, CacheConfiguration> allCaches() {
Map<String, CacheConfiguration> cfgs = new HashMap<>();
@@ -403,6 +434,10 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
return cfgs;
}
+ /**
+ * @return Collection of all caches start requests.
+ * @throws IgniteCheckedException If failed to create requests.
+ */
private List<DynamicCacheChangeRequest> startAllCachesRequests() throws IgniteCheckedException {
assert !ctx.config().isDaemon();
@@ -430,6 +465,27 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
}
}
+ /**
+ * @return Collection of requests to stop caches.
+ */
+ private List<DynamicCacheChangeRequest> stopAllCachesRequests() {
+ Collection<CacheConfiguration> cacheCfgs = allCaches().values();
+
+ List<DynamicCacheChangeRequest> reqs = new ArrayList<>(cacheCfgs.size());
+
+ for (CacheConfiguration cfg : cacheCfgs) {
+ DynamicCacheChangeRequest req = stopRequest(ctx, cfg.getName(), false, false);
+
+ reqs.add(req);
+ }
+
+ return reqs;
+ }
+
+ /**
+ * @param cfg Configuration to create request for.
+ * @return Dynamic cache change request.
+ */
private DynamicCacheChangeRequest createRequest(CacheConfiguration cfg) {
assert cfg != null;
assert cfg.getName() != null;
@@ -449,20 +505,6 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
return req;
}
- private List<DynamicCacheChangeRequest> stopAllCachesRequests() {
- Collection<CacheConfiguration> cacheCfgs = allCaches().values();
-
- List<DynamicCacheChangeRequest> reqs = new ArrayList<>(cacheCfgs.size());
-
- for (CacheConfiguration cfg : cacheCfgs) {
- DynamicCacheChangeRequest req = stopRequest(ctx, cfg.getName(), false, false);
-
- reqs.add(req);
- }
-
- return reqs;
- }
-
/**
*
*/
@@ -478,30 +520,29 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
return globalState == ACTIVE;
}
- @Override public void onKernalStart() throws IgniteCheckedException {
- super.onKernalStart();
-
- // First node started (coordinator).
- if (ctx.discovery().serverNodes(AffinityTopologyVersion.NONE).get(0).isLocal())
- cacheData.putAll(localCacheData.caches());
- }
-
- private final ConcurrentHashMap<String, CacheInfo> cacheData = new ConcurrentHashMap<>();
-
- private volatile CacheJoinNodeDiscoveryData localCacheData;
-
- public boolean isLocalConfigure(String cacheName){
+ /**
+ * @param cacheName Cache name to check.
+ * @return Locally configured flag.
+ */
+ public boolean isLocallyConfigured(String cacheName){
assert localCacheData != null;
return localCacheData.caches().containsKey(cacheName) || localCacheData.templates().containsKey(cacheName);
}
- // Invoke if cluster inactive.
+ /**
+ * Invoked if cluster is inactive.
+ *
+ * @param dataBag Bag to collect data to.
+ */
public void collectGridNodeData0(DiscoveryDataBag dataBag) {
if (!dataBag.commonDataCollectedFor(CACHE_PROC.ordinal()))
dataBag.addGridCommonData(CACHE_PROC.ordinal(), cacheData);
}
+ /**
+ * @param data Joining node discovery data.
+ */
public void onJoiningNodeDataReceived0(JoiningNodeDiscoveryData data) {
if (data.hasJoiningNodeData()) {
if (data.joiningNodeData() instanceof CacheJoinNodeDiscoveryData) {
@@ -624,7 +665,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
for (Map.Entry<UUID, Exception> entry : exs.entrySet())
e.addSuppressed(entry.getValue());
- log.error("Fail while revert activation request changes", e);
+ U.error(log, "Failed to revert activation request changes", e);
}
}
else {
@@ -652,7 +693,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
final boolean client = ctx.clientNode();
if (log.isInfoEnabled())
- log.info("Start activation process [nodeId=" + this.ctx.localNodeId() + ", client=" + client +
+ log.info("Start activation process [nodeId=" + ctx.localNodeId() + ", client=" + client +
", topVer=" + cgsCtx.topVer + "]");
Collection<StoredCacheData> cfgs = new ArrayList<>();
@@ -677,13 +718,13 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
sharedCtx.database().onActivate(ctx);
if (log.isInfoEnabled())
- log.info("Success activate wal, dataBase, pageStore [nodeId="
+ log.info("Successfully activated persistence managers [nodeId="
+ ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]");
return null;
}
catch (Exception e) {
- log.error("Fail activate wal, dataBase, pageStore [nodeId=" + ctx.localNodeId() + ", client=" + client +
+ U.error(log, "Failed to activate persistence managers [nodeId=" + ctx.localNodeId() + ", client=" + client +
", topVer=" + cgsCtx.topVer + "]", e);
if (!client)
@@ -700,7 +741,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
final boolean client = ctx.clientNode();
if (log.isInfoEnabled())
- log.info("Start deactivate process [id=" + ctx.localNodeId() + ", client=" +
+ log.info("Starting deactivation [id=" + ctx.localNodeId() + ", client=" +
client + ", topVer=" + cgsCtx.topVer + "]");
try {
@@ -709,13 +750,13 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
ctx.service().onDeActivate(ctx);
if (log.isInfoEnabled())
- log.info("Success deactivate services, dataStructures, database, pageStore, wal [id=" + ctx.localNodeId() + ", client=" +
+ log.info("Successfully deactivated persistence processors [id=" + ctx.localNodeId() + ", client=" +
client + ", topVer=" + cgsCtx.topVer + "]");
return null;
}
catch (Exception e) {
- log.error("DeActivation fail [nodeId=" + ctx.localNodeId() + ", client=" + client +
+ U.error(log, "Failed to execute deactivation callback [nodeId=" + ctx.localNodeId() + ", client=" + client +
", topVer=" + cgsCtx.topVer + "]", e);
return e;
@@ -743,21 +784,21 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
ctx.dataStructures().onActivate(ctx);
if (log.isInfoEnabled())
- log.info("Success final activate [nodeId="
+ log.info("Successfully performed final activation steps [nodeId="
+ ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]");
}
catch (Exception ex) {
e = ex;
- log.error("Fail activate finished [nodeId=" + ctx.localNodeId() + ", client=" + client +
- ", topVer=" + GridClusterStateProcessor.this.lastCgsCtx.topVer + "]", ex);
+ U.error(log, "Failed to perform final activation steps [nodeId=" + ctx.localNodeId() +
+ ", client=" + client + ", topVer=" + lastCgsCtx.topVer + "]", ex);
}
finally {
globalState = ACTIVE;
sendChangeGlobalStateResponse(cgsCtx.requestId, cgsCtx.initiatingNodeId, e);
- GridClusterStateProcessor.this.lastCgsCtx = null;
+ lastCgsCtx = null;
}
}
});
@@ -772,7 +813,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
final boolean client = ctx.clientNode();
if (log.isInfoEnabled())
- log.info("Success final deactivate [nodeId="
+ log.info("Successfully performed final deactivation steps [nodeId="
+ ctx.localNodeId() + ", client=" + client + ", topVer=" + cgsCtx.topVer + "]");
Exception ex = null;
@@ -797,7 +838,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
sendChangeGlobalStateResponse(cgsCtx.requestId, cgsCtx.initiatingNodeId, ex);
- this.lastCgsCtx = null;
+ lastCgsCtx = null;
}
/**
@@ -830,7 +871,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
GridChangeGlobalStateMessageResponse actResp = new GridChangeGlobalStateMessageResponse(requestId, ex);
if (log.isDebugEnabled())
- log.debug("Send change global state response [nodeId=" + ctx.localNodeId() +
+ log.debug("Sending global state change response [nodeId=" + ctx.localNodeId() +
", topVer=" + ctx.discovery().topologyVersionEx() + ", response=" + actResp + "]");
if (ctx.localNodeId().equals(initNodeId))
@@ -893,7 +934,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/**
*
*/
- private static class GridChangeGlobalStateFuture extends GridFutureAdapter {
+ private static class GridChangeGlobalStateFuture extends GridFutureAdapter<Void> {
/** Request id. */
@GridToStringInclude
private final UUID requestId;
@@ -919,7 +960,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
/** */
@GridToStringInclude
- private final GridFutureAdapter initFut = new GridFutureAdapter();
+ private final GridFutureAdapter<?> initFut = new GridFutureAdapter<>();
/** Grid logger. */
@GridToStringExclude
@@ -932,7 +973,8 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
this.requestId = requestId;
this.activate = activate;
this.ctx = ctx;
- this.log = ctx.log(getClass());
+
+ log = ctx.log(getClass());
}
/**
@@ -1025,7 +1067,7 @@ public class GridClusterStateProcessor extends GridProcessorAdapter {
}
/** {@inheritDoc} */
- @Override public boolean onDone(@Nullable Object res, @Nullable Throwable err) {
+ @Override public boolean onDone(@Nullable Void res, @Nullable Throwable err) {
ctx.state().cgsLocFut.set(null);
return super.onDone(res, err);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index 8ae6db8..6874bbe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -65,7 +65,7 @@ import org.apache.ignite.internal.managers.communication.GridIoPolicy;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.datastreamer.DataStreamerCacheUpdaters;
import org.apache.ignite.internal.processors.igfs.data.IgfsDataPutProcessor;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
index b3fa47c..3a445bf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/schema/SchemaIndexCacheVisitorImpl.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearCacheAdapter;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java
index da2203d..f842440 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/ClusterNodeMetricsSelfTest.java
@@ -29,7 +29,7 @@ import org.apache.ignite.cluster.ClusterMetrics;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.events.Event;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
import org.apache.ignite.lang.IgnitePredicate;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
index 93b68f3..02c521e 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoLoadSelfTest.java
@@ -32,8 +32,8 @@ import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
index a8b0c8b..7b420cc 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheGroupsTest.java
@@ -78,7 +78,7 @@ import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.binary.BinaryMarshaller;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicyFactory;
import org.apache.ignite.internal.util.lang.GridAbsPredicate;
[43/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
deleted file mode 100644
index 98204eb..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/BPlusTree.java
+++ /dev/null
@@ -1,4808 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree;
-
-import java.io.Externalizable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.FixLeftmostChildRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.FixRemoveId;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageAddRootRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord;
-import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.util.GridArrays;
-import org.apache.ignite.internal.util.GridLongList;
-import org.apache.ignite.internal.util.IgniteTree;
-import org.apache.ignite.internal.util.lang.GridCursor;
-import org.apache.ignite.internal.util.lang.GridTreePrinter;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteInClosure;
-
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Bool.DONE;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Bool.FALSE;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Bool.READY;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Bool.TRUE;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Result.FOUND;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Result.GO_DOWN;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Result.GO_DOWN_X;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Result.NOT_FOUND;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Result.RETRY;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.Result.RETRY_ROOT;
-
-/**
- * Abstract B+Tree.
- */
-@SuppressWarnings({"RedundantThrowsDeclaration", "ConstantValueVariableUse"})
-public abstract class BPlusTree<L, T extends L> extends DataStructure implements IgniteTree<L, T> {
- /** */
- private static final Object[] EMPTY = {};
-
- /** */
- private static volatile boolean interrupted;
-
- /** */
- private final AtomicBoolean destroyed = new AtomicBoolean(false);
-
- /** */
- private final String name;
-
- /** */
- private final float minFill;
-
- /** */
- private final float maxFill;
-
- /** */
- protected final long metaPageId;
-
- /** */
- private boolean canGetRowFromInner;
-
- /** */
- private IOVersions<? extends BPlusInnerIO<L>> innerIos;
-
- /** */
- private IOVersions<? extends BPlusLeafIO<L>> leafIos;
-
- /** */
- private final AtomicLong globalRmvId;
-
- /** */
- private volatile TreeMetaData treeMeta;
-
- /** */
- private final GridTreePrinter<Long> treePrinter = new GridTreePrinter<Long>() {
- /** */
- private boolean keys = true;
-
- @Override protected List<Long> getChildren(final Long pageId) {
- if (pageId == null || pageId == 0L)
- return null;
-
- try {
- long page = acquirePage(pageId);
-
- try {
- long pageAddr = readLock(pageId, page); // No correctness guaranties.
-
- try {
- BPlusIO io = io(pageAddr);
-
- if (io.isLeaf())
- return null;
-
- int cnt = io.getCount(pageAddr);
-
- assert cnt >= 0 : cnt;
-
- List<Long> res;
-
- if (cnt > 0) {
- res = new ArrayList<>(cnt + 1);
-
- for (int i = 0; i < cnt; i++)
- res.add(inner(io).getLeft(pageAddr, i));
-
- res.add(inner(io).getRight(pageAddr, cnt - 1));
- }
- else {
- long left = inner(io).getLeft(pageAddr, 0);
-
- res = left == 0 ? Collections.<Long>emptyList() : Collections.singletonList(left);
- }
-
- return res;
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
- catch (IgniteCheckedException ignored) {
- throw new AssertionError("Can not acquire page.");
- }
- }
-
- @Override protected String formatTreeNode(final Long pageId) {
- if (pageId == null)
- return ">NPE<";
-
- if (pageId == 0L)
- return "<Zero>";
-
- try {
- long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page); // No correctness guaranties.
- try {
- BPlusIO<L> io = io(pageAddr);
-
- return printPage(io, pageAddr, keys);
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
- catch (IgniteCheckedException e) {
- throw new IllegalStateException(e);
- }
- }
- };
-
- /** */
- private final GetPageHandler<Get> askNeighbor = new AskNeighbor();
-
- /**
- *
- */
- private class AskNeighbor extends GetPageHandler<Get> {
- /** {@inheritDoc} */
- @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Get g, int isBack) {
- assert !io.isLeaf(); // Inner page.
-
- boolean back = isBack == TRUE.ordinal();
-
- long res = doAskNeighbor(io, pageAddr, back);
-
- if (back) {
- if (io.getForward(pageAddr) != g.backId) // See how g.backId is setup in removeDown for this check.
- return RETRY;
-
- g.backId(res);
- }
- else {
- assert isBack == FALSE.ordinal() : isBack;
-
- g.fwdId(res);
- }
-
- return FOUND;
- }
- }
-
- /** */
- private final GetPageHandler<Get> search = new Search();
-
- /**
- *
- */
- private class Search extends GetPageHandler<Get> {
- /** {@inheritDoc} */
- @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Get g, int lvl)
- throws IgniteCheckedException {
- // Check the triangle invariant.
- if (io.getForward(pageAddr) != g.fwdId)
- return RETRY;
-
- boolean needBackIfRouting = g.backId != 0;
-
- g.backId(0L); // Usually we'll go left down and don't need it.
-
- int cnt = io.getCount(pageAddr);
-
- int idx;
-
- if (g.findLast)
- idx = io.isLeaf() ? cnt - 1 : -cnt - 1; // (-cnt - 1) mimics not_found result of findInsertionPoint
- // in case of cnt = 0 we end up in 'not found' branch below with idx being 0 after fix() adjustment
- else
- idx = findInsertionPoint(lvl, io, pageAddr, 0, cnt, g.row, g.shift);
-
- boolean found = idx >= 0;
-
- if (found) { // Found exact match.
- assert g.getClass() != GetCursor.class;
-
- if (g.found(io, pageAddr, idx, lvl))
- return FOUND;
-
- // Else we need to reach leaf page, go left down.
- }
- else {
- idx = fix(idx);
-
- if (g.notFound(io, pageAddr, idx, lvl)) // No way down, stop here.
- return NOT_FOUND;
- }
-
- assert !io.isLeaf() : io;
-
- // If idx == cnt then we go right down, else left down: getLeft(cnt) == getRight(cnt - 1).
- g.pageId(inner(io).getLeft(pageAddr, idx));
-
- // If we see the tree in consistent state, then our right down page must be forward for our left down page,
- // we need to setup fwdId and/or backId to be able to check this invariant on lower level.
- if (idx < cnt) {
- // Go left down here.
- g.fwdId(inner(io).getRight(pageAddr, idx));
- }
- else {
- // Go right down here or it is an empty branch.
- assert idx == cnt;
-
- // Here child's forward is unknown to us (we either go right or it is an empty "routing" page),
- // need to ask our forward about the child's forward (it must be leftmost child of our forward page).
- // This is ok from the locking standpoint because we take all locks in the forward direction.
- long fwdId = io.getForward(pageAddr);
-
- // Setup fwdId.
- if (fwdId == 0)
- g.fwdId(0L);
- else {
- // We can do askNeighbor on forward page here because we always take locks in forward direction.
- Result res = askNeighbor(fwdId, g, false);
-
- if (res != FOUND)
- return res; // Retry.
- }
-
- // Setup backId.
- if (cnt != 0) // It is not a routing page and we are going to the right, can get backId here.
- g.backId(inner(io).getLeft(pageAddr, cnt - 1));
- else if (needBackIfRouting) {
- // Can't get backId here because of possible deadlock and it is only needed for remove operation.
- return GO_DOWN_X;
- }
- }
-
- return GO_DOWN;
- }
- }
-
- /** */
- private final GetPageHandler<Put> replace = new Replace();
-
- /**
- *
- */
- private class Replace extends GetPageHandler<Put> {
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Put p, int lvl)
- throws IgniteCheckedException {
- // Check the triangle invariant.
- if (io.getForward(pageAddr) != p.fwdId)
- return RETRY;
-
- assert p.btmLvl == 0 : "split is impossible with replace";
-
- final int cnt = io.getCount(pageAddr);
- final int idx = findInsertionPoint(lvl, io, pageAddr, 0, cnt, p.row, 0);
-
- if (idx < 0) // Not found, split or merge happened.
- return RETRY;
-
- // Replace link at idx with new one.
- // Need to read link here because `p.finish()` will clear row.
- L newRow = p.row;
-
- // Detach the old row if we are on a leaf page.
- if (lvl == 0) {
- assert p.oldRow == null; // The old row must be set only once.
-
- // Inner replace state must be consistent by the end of the operation.
- assert p.needReplaceInner == FALSE || p.needReplaceInner == DONE : p.needReplaceInner;
-
- // Need to replace inner key if now we are replacing the rightmost row and have a forward page.
- if (canGetRowFromInner && idx + 1 == cnt && p.fwdId != 0L && p.needReplaceInner == FALSE) {
- // Can happen only for invoke, otherwise inner key must be replaced on the way down.
- assert p.invoke != null;
-
- // We need to restart the operation from root to perform inner replace.
- // On the second pass we will not get here (will avoid infinite loop) because
- // needReplaceInner will be DONE or our key will not be the rightmost anymore.
- return RETRY_ROOT;
- }
-
- // Get old row in leaf page to reduce contention at upper level.
- p.oldRow = p.needOld ? getRow(io, pageAddr, idx) : (T)Boolean.TRUE;
-
- p.finish();
- }
-
- boolean needWal = needWalDeltaRecord(pageId, page, null);
-
- byte[] newRowBytes = io.store(pageAddr, idx, newRow, null, needWal);
-
- if (needWal)
- wal.log(new ReplaceRecord<>(cacheId, pageId, io, newRowBytes, idx));
-
- return FOUND;
- }
- }
-
- /** */
- private final GetPageHandler<Put> insert = new Insert();
-
- /**
- *
- */
- private class Insert extends GetPageHandler<Put> {
- /** {@inheritDoc} */
- @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Put p, int lvl)
- throws IgniteCheckedException {
- assert p.btmLvl == lvl : "we must always insert at the bottom level: " + p.btmLvl + " " + lvl;
-
- // Check triangle invariant.
- if (io.getForward(pageAddr) != p.fwdId)
- return RETRY;
-
- int cnt = io.getCount(pageAddr);
- int idx = findInsertionPoint(lvl, io, pageAddr, 0, cnt, p.row, 0);
-
- if (idx >= 0) // We do not support concurrent put of the same key.
- throw new IllegalStateException("Duplicate row in index.");
-
- idx = fix(idx);
-
- // Do insert.
- L moveUpRow = p.insert(pageId, page, pageAddr, io, idx, lvl);
-
- // Check if split happened.
- if (moveUpRow != null) {
- p.btmLvl++; // Get high.
- p.row = moveUpRow;
-
- if (p.invoke != null)
- p.invoke.row = moveUpRow;
-
- // Here forward page can't be concurrently removed because we keep write lock on tail which is the only
- // page who knows about the forward page, because it was just produced by split.
- p.rightId = io.getForward(pageAddr);
- p.tail(pageId, page, pageAddr);
-
- assert p.rightId != 0;
- }
- else
- p.finish();
-
- return FOUND;
- }
- }
-
- /** */
- private final GetPageHandler<Remove> rmvFromLeaf = new RemoveFromLeaf();
-
- /**
- *
- */
- private class RemoveFromLeaf extends GetPageHandler<Remove> {
- /** {@inheritDoc} */
- @Override public Result run0(long leafId, long leafPage, long leafAddr, BPlusIO<L> io, Remove r, int lvl)
- throws IgniteCheckedException {
- assert lvl == 0 : lvl; // Leaf.
-
- // Check the triangle invariant.
- if (io.getForward(leafAddr) != r.fwdId)
- return RETRY;
-
- final int cnt = io.getCount(leafAddr);
-
- assert cnt <= Short.MAX_VALUE: cnt;
-
- int idx = findInsertionPoint(lvl, io, leafAddr, 0, cnt, r.row, 0);
-
- if (idx < 0)
- return RETRY; // We've found exact match on search but now it's gone.
-
- assert idx >= 0 && idx < cnt: idx;
-
- // Need to do inner replace when we remove the rightmost element and the leaf have no forward page,
- // i.e. it is not the rightmost leaf of the tree.
- boolean needReplaceInner = canGetRowFromInner && idx == cnt - 1 && io.getForward(leafAddr) != 0;
-
- // !!! Before modifying state we have to make sure that we will not go for retry.
-
- // We may need to replace inner key or want to merge this leaf with sibling after the remove -> keep lock.
- if (needReplaceInner ||
- // We need to make sure that we have back or forward to be able to merge.
- ((r.fwdId != 0 || r.backId != 0) && mayMerge(cnt - 1, io.getMaxCount(leafAddr, pageSize())))) {
- // If we have backId then we've already locked back page, nothing to do here.
- if (r.fwdId != 0 && r.backId == 0) {
- Result res = r.lockForward(0);
-
- if (res != FOUND) {
- assert r.tail == null;
-
- return res; // Retry.
- }
-
- assert r.tail != null; // We've just locked forward page.
- }
-
- // Retry must reset these fields when we release the whole branch without remove.
- assert r.needReplaceInner == FALSE: "needReplaceInner";
- assert r.needMergeEmptyBranch == FALSE: "needMergeEmptyBranch";
-
- if (cnt == 1) // It was the last element on the leaf.
- r.needMergeEmptyBranch = TRUE;
-
- if (needReplaceInner)
- r.needReplaceInner = TRUE;
-
- Tail<L> t = r.addTail(leafId, leafPage, leafAddr, io, 0, Tail.EXACT);
-
- t.idx = (short)idx;
-
- // We will do the actual remove only when we made sure that
- // we've locked the whole needed branch correctly.
- return FOUND;
- }
-
- r.removeDataRowFromLeaf(leafId, leafPage, leafAddr, null, io, cnt, idx);
-
- return FOUND;
- }
- }
-
- /** */
- private final GetPageHandler<Remove> lockBackAndRmvFromLeaf = new LockBackAndRmvFromLeaf();
-
- /**
- *
- */
- private class LockBackAndRmvFromLeaf extends GetPageHandler<Remove> {
- /** {@inheritDoc} */
- @Override protected Result run0(long backId, long backPage, long backAddr, BPlusIO<L> io, Remove r, int lvl)
- throws IgniteCheckedException {
- // Check that we have consistent view of the world.
- if (io.getForward(backAddr) != r.pageId)
- return RETRY;
-
- // Correct locking order: from back to forward.
- Result res = r.doRemoveFromLeaf();
-
- // Keep locks on back and leaf pages for subsequent merges.
- if (res == FOUND && r.tail != null)
- r.addTail(backId, backPage, backAddr, io, lvl, Tail.BACK);
-
- return res;
- }
- }
-
- /** */
- private final GetPageHandler<Remove> lockBackAndTail = new LockBackAndTail();
-
- /**
- *
- */
- private class LockBackAndTail extends GetPageHandler<Remove> {
- /** {@inheritDoc} */
- @Override public Result run0(long backId, long backPage, long backAddr, BPlusIO<L> io, Remove r, int lvl)
- throws IgniteCheckedException {
- // Check that we have consistent view of the world.
- if (io.getForward(backAddr) != r.pageId)
- return RETRY;
-
- // Correct locking order: from back to forward.
- Result res = r.doLockTail(lvl);
-
- if (res == FOUND)
- r.addTail(backId, backPage, backAddr, io, lvl, Tail.BACK);
-
- return res;
- }
- }
-
- /** */
- private final GetPageHandler<Remove> lockTailForward = new LockTailForward();
-
- /**
- *
- */
- private class LockTailForward extends GetPageHandler<Remove> {
- /** {@inheritDoc} */
- @Override protected Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Remove r, int lvl)
- throws IgniteCheckedException {
- r.addTail(pageId, page, pageAddr, io, lvl, Tail.FORWARD);
-
- return FOUND;
- }
- }
-
- /** */
- private final GetPageHandler<Remove> lockTail = new LockTail();
-
- /**
- *
- */
- private class LockTail extends GetPageHandler<Remove> {
- /** {@inheritDoc} */
- @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Remove r, int lvl)
- throws IgniteCheckedException {
- assert lvl > 0 : lvl; // We are not at the bottom.
-
- // Check that we have a correct view of the world.
- if (io.getForward(pageAddr) != r.fwdId)
- return RETRY;
-
- // We don't have a back page, need to lock our forward and become a back for it.
- if (r.fwdId != 0 && r.backId == 0) {
- Result res = r.lockForward(lvl);
-
- if (res != FOUND)
- return res; // Retry.
- }
-
- r.addTail(pageId, page, pageAddr, io, lvl, Tail.EXACT);
-
- return FOUND;
- }
- }
-
- /** */
- private final PageHandler<Void, Bool> cutRoot = new CutRoot();
-
- /**
- *
- */
- private class CutRoot extends PageHandler<Void, Bool> {
- /** {@inheritDoc} */
- @Override public Bool run(int cacheId, long metaId, long metaPage, long metaAddr, PageIO iox, Boolean walPlc, Void ignore, int lvl)
- throws IgniteCheckedException {
- // Safe cast because we should never recycle meta page until the tree is destroyed.
- BPlusMetaIO io = (BPlusMetaIO)iox;
-
- assert lvl == io.getRootLevel(metaAddr); // Can drop only root.
-
- io.cutRoot(metaAddr, pageSize());
-
- if (needWalDeltaRecord(metaId, metaPage, walPlc))
- wal.log(new MetaPageCutRootRecord(cacheId, metaId));
-
- int newLvl = lvl - 1;
-
- assert io.getRootLevel(metaAddr) == newLvl;
-
- treeMeta = new TreeMetaData(newLvl, io.getFirstPageId(metaAddr, newLvl));
-
- return TRUE;
- }
- }
-
- /** */
- private final PageHandler<Long, Bool> addRoot = new AddRoot();
-
- /**
- *
- */
- private class AddRoot extends PageHandler<Long, Bool> {
- /** {@inheritDoc} */
- @Override public Bool run(int cacheId, long metaId, long metaPage, long pageAddr, PageIO iox, Boolean walPlc, Long rootPageId, int lvl)
- throws IgniteCheckedException {
- assert rootPageId != null;
-
- // Safe cast because we should never recycle meta page until the tree is destroyed.
- BPlusMetaIO io = (BPlusMetaIO)iox;
-
- assert lvl == io.getLevelsCount(pageAddr);
-
- io.addRoot(pageAddr, rootPageId, pageSize());
-
- if (needWalDeltaRecord(metaId, metaPage, walPlc))
- wal.log(new MetaPageAddRootRecord(cacheId, metaId, rootPageId));
-
- assert io.getRootLevel(pageAddr) == lvl;
- assert io.getFirstPageId(pageAddr, lvl) == rootPageId;
-
- treeMeta = new TreeMetaData(lvl, rootPageId);
-
- return TRUE;
- }
- }
-
- /** */
- private final PageHandler<Long, Bool> initRoot = new InitRoot();
-
- /**
- *
- */
- private class InitRoot extends PageHandler<Long, Bool> {
- /** {@inheritDoc} */
- @Override public Bool run(int cacheId, long metaId, long metaPage, long pageAddr, PageIO iox, Boolean walPlc, Long rootId, int inlineSize)
- throws IgniteCheckedException {
- assert rootId != null;
-
- // Safe cast because we should never recycle meta page until the tree is destroyed.
- BPlusMetaIO io = (BPlusMetaIO)iox;
-
- io.initRoot(pageAddr, rootId, pageSize());
- io.setInlineSize(pageAddr, inlineSize);
-
- if (needWalDeltaRecord(metaId, metaPage, walPlc))
- wal.log(new MetaPageInitRootInlineRecord(cacheId, metaId, rootId, inlineSize));
-
- assert io.getRootLevel(pageAddr) == 0;
- assert io.getFirstPageId(pageAddr, 0) == rootId;
-
- treeMeta = new TreeMetaData(0, rootId);
-
- return TRUE;
- }
- }
-
- /**
- * @param name Tree name.
- * @param cacheId Cache ID.
- * @param pageMem Page memory.
- * @param wal Write ahead log manager.
- * @param globalRmvId Remove ID.
- * @param metaPageId Meta page ID.
- * @param reuseList Reuse list.
- * @param innerIos Inner IO versions.
- * @param leafIos Leaf IO versions.
- * @throws IgniteCheckedException If failed.
- */
- protected BPlusTree(
- String name,
- int cacheId,
- PageMemory pageMem,
- IgniteWriteAheadLogManager wal,
- AtomicLong globalRmvId,
- long metaPageId,
- ReuseList reuseList,
- IOVersions<? extends BPlusInnerIO<L>> innerIos,
- IOVersions<? extends BPlusLeafIO<L>> leafIos
- ) throws IgniteCheckedException {
- this(name, cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList);
- setIos(innerIos, leafIos);
- }
-
- /**
- * @param name Tree name.
- * @param cacheId Cache ID.
- * @param pageMem Page memory.
- * @param wal Write ahead log manager.
- * @param globalRmvId Remove ID.
- * @param metaPageId Meta page ID.
- * @param reuseList Reuse list.
- * @throws IgniteCheckedException If failed.
- */
- protected BPlusTree(
- String name,
- int cacheId,
- PageMemory pageMem,
- IgniteWriteAheadLogManager wal,
- AtomicLong globalRmvId,
- long metaPageId,
- ReuseList reuseList
- ) throws IgniteCheckedException {
- super(cacheId, pageMem, wal);
-
- assert !F.isEmpty(name);
-
- // TODO make configurable: 0 <= minFill <= maxFill <= 1
- minFill = 0f; // Testing worst case when merge happens only on empty page.
- maxFill = 0f; // Avoiding random effects on testing.
-
- assert metaPageId != 0L;
-
- this.metaPageId = metaPageId;
- this.name = name;
- this.reuseList = reuseList;
- this.globalRmvId = globalRmvId;
- }
-
- /**
- * @param innerIos Inner IO versions.
- * @param leafIos Leaf IO versions.
- */
- protected void setIos(IOVersions<? extends BPlusInnerIO<L>> innerIos,
- IOVersions<? extends BPlusLeafIO<L>> leafIos) {
- assert innerIos != null;
- assert leafIos != null;
-
- this.canGetRowFromInner = innerIos.latest().canGetRow(); // TODO refactor
- this.innerIos = innerIos;
- this.leafIos = leafIos;
- }
-
- /**
- * @return Tree name.
- */
- public final String getName() {
- return name;
- }
-
- /**
- * Initialize new tree.
- *
- * @param initNew {@code True} if new tree should be created.
- * @throws IgniteCheckedException If failed.
- */
- protected final void initTree(boolean initNew) throws IgniteCheckedException {
- initTree(initNew, 0);
- }
-
- /**
- * Initialize new tree.
- *
- * @param initNew {@code True} if new tree should be created.
- * @param inlineSize Inline size.
- * @throws IgniteCheckedException If failed.
- */
- protected final void initTree(boolean initNew, int inlineSize) throws IgniteCheckedException {
- if (initNew) {
- // Allocate the first leaf page, it will be our root.
- long rootId = allocatePage(null);
-
- init(rootId, latestLeafIO());
-
- // Initialize meta page with new root page.
- Bool res = write(metaPageId, initRoot, BPlusMetaIO.VERSIONS.latest(), rootId, inlineSize, FALSE);
-
- assert res == TRUE: res;
-
- assert treeMeta != null;
- }
- }
-
- /**
- * @return Tree meta data.
- * @throws IgniteCheckedException If failed.
- */
- private TreeMetaData treeMeta() throws IgniteCheckedException {
- TreeMetaData meta0 = treeMeta;
-
- if (meta0 != null)
- return meta0;
-
- final long metaPage = acquirePage(metaPageId);
- try {
- long pageAddr = readLock(metaPageId, metaPage); // Meta can't be removed.
-
- assert pageAddr != 0 : "Failed to read lock meta page [metaPageId=" +
- U.hexLong(metaPageId) + ']';
-
- try {
- BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(pageAddr);
-
- int rootLvl = io.getRootLevel(pageAddr);
- long rootId = io.getFirstPageId(pageAddr, rootLvl);
-
- treeMeta = meta0 = new TreeMetaData(rootLvl, rootId);
- }
- finally {
- readUnlock(metaPageId, metaPage, pageAddr);
- }
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
-
- return meta0;
- }
-
- /**
- * @return Root level.
- * @throws IgniteCheckedException If failed.
- */
- private int getRootLevel() throws IgniteCheckedException {
- TreeMetaData meta0 = treeMeta();
-
- assert meta0 != null;
-
- return meta0.rootLvl;
- }
-
- /**
- * @param metaId Meta page ID.
- * @param metaPage Meta page pointer.
- * @param lvl Level, if {@code 0} then it is a bottom level, if negative then root.
- * @return Page ID.
- */
- private long getFirstPageId(long metaId, long metaPage, int lvl) {
- long pageAddr = readLock(metaId, metaPage); // Meta can't be removed.
-
- try {
- BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(pageAddr);
-
- if (lvl < 0)
- lvl = io.getRootLevel(pageAddr);
-
- if (lvl >= io.getLevelsCount(pageAddr))
- return 0;
-
- return io.getFirstPageId(pageAddr, lvl);
- }
- finally {
- readUnlock(metaId, metaPage, pageAddr);
- }
- }
-
- /**
- * @param upper Upper bound.
- * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row.
- * @return Cursor.
- * @throws IgniteCheckedException If failed.
- */
- private GridCursor<T> findLowerUnbounded(L upper, Object x) throws IgniteCheckedException {
- ForwardCursor cursor = new ForwardCursor(null, upper, x);
-
- long firstPageId;
-
- long metaPage = acquirePage(metaPageId);
- try {
- firstPageId = getFirstPageId(metaPageId, metaPage, 0); // Level 0 is always at the bottom.
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
-
- long firstPage = acquirePage(firstPageId);
-
- try {
- long pageAddr = readLock(firstPageId, firstPage); // We always merge pages backwards, the first page is never removed.
-
- try {
- cursor.init(pageAddr, io(pageAddr), 0);
- }
- finally {
- readUnlock(firstPageId, firstPage, pageAddr);
- }
- }
- finally {
- releasePage(firstPageId, firstPage);
- }
-
- return cursor;
- }
-
- /**
- * Check if the tree is getting destroyed.
- */
- private void checkDestroyed() {
- if (destroyed.get())
- throw new IllegalStateException("Tree is being concurrently destroyed: " + getName());
- }
-
- /**
- * @param lower Lower bound inclusive or {@code null} if unbounded.
- * @param upper Upper bound inclusive or {@code null} if unbounded.
- * @return Cursor.
- * @throws IgniteCheckedException If failed.
- */
- @Override public GridCursor<T> find(L lower, L upper) throws IgniteCheckedException {
- return find(lower, upper, null);
- }
-
- /**
- * @param lower Lower bound inclusive or {@code null} if unbounded.
- * @param upper Upper bound inclusive or {@code null} if unbounded.
- * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row.
- * @return Cursor.
- * @throws IgniteCheckedException If failed.
- */
- public final GridCursor<T> find(L lower, L upper, Object x) throws IgniteCheckedException {
- checkDestroyed();
-
- try {
- if (lower == null)
- return findLowerUnbounded(upper, x);
-
- ForwardCursor cursor = new ForwardCursor(lower, upper, x);
-
- cursor.find();
-
- return cursor;
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
- }
- finally {
- checkDestroyed();
- }
- }
-
- /** {@inheritDoc} */
- @Override public T findFirst() throws IgniteCheckedException {
- checkDestroyed();
-
- try {
- long firstPageId;
-
- long metaPage = acquirePage(metaPageId);
- try {
- firstPageId = getFirstPageId(metaPageId, metaPage, 0);
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
-
- long page = acquirePage(firstPageId);
-
- try {
- long pageAddr = readLock(firstPageId, page);
-
- try {
- BPlusIO<L> io = io(pageAddr);
-
- int cnt = io.getCount(pageAddr);
-
- if (cnt == 0)
- return null;
-
- return getRow(io, pageAddr, 0);
- }
- finally {
- readUnlock(firstPageId, page, pageAddr);
- }
- }
- finally {
- releasePage(firstPageId, page);
- }
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on first row lookup", e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on first row lookup", e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on first row lookup", e);
- }
- finally {
- checkDestroyed();
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override public T findLast() throws IgniteCheckedException {
- checkDestroyed();
-
- try {
- GetOne g = new GetOne(null, null, true);
- doFind(g);
-
- return (T)g.row;
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on last row lookup", e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on last row lookup", e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on last row lookup", e);
- }
- finally {
- checkDestroyed();
- }
- }
-
- /**
- * @param row Lookup row for exact match.
- * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row.
- * @return Found result or {@code null}.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- public final <R> R findOne(L row, Object x) throws IgniteCheckedException {
- checkDestroyed();
-
- try {
- GetOne g = new GetOne(row, x, false);
-
- doFind(g);
-
- return (R)g.row;
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on lookup row: " + row, e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on lookup row: " + row, e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on lookup row: " + row, e);
- }
- finally {
- checkDestroyed();
- }
- }
-
- /**
- * @param row Lookup row for exact match.
- * @return Found row.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- @Override public final T findOne(L row) throws IgniteCheckedException {
- return findOne(row, null);
- }
-
- /**
- * @param g Get.
- * @throws IgniteCheckedException If failed.
- */
- private void doFind(Get g) throws IgniteCheckedException {
- for (;;) { // Go down with retries.
- g.init();
-
- switch (findDown(g, g.rootId, 0L, g.rootLvl)) {
- case RETRY:
- case RETRY_ROOT:
- checkInterrupted();
-
- continue;
-
- default:
- return;
- }
- }
- }
-
- /**
- * @param g Get.
- * @param pageId Page ID.
- * @param fwdId Expected forward page ID.
- * @param lvl Level.
- * @return Result code.
- * @throws IgniteCheckedException If failed.
- */
- private Result findDown(final Get g, final long pageId, final long fwdId, final int lvl)
- throws IgniteCheckedException {
- long page = acquirePage(pageId);
-
- try {
- for (;;) {
- // Init args.
- g.pageId = pageId;
- g.fwdId = fwdId;
-
- Result res = read(pageId, page, search, g, lvl, RETRY);
-
- switch (res) {
- case GO_DOWN:
- case GO_DOWN_X:
- assert g.pageId != pageId;
- assert g.fwdId != fwdId || fwdId == 0;
-
- // Go down recursively.
- res = findDown(g, g.pageId, g.fwdId, lvl - 1);
-
- switch (res) {
- case RETRY:
- checkInterrupted();
-
- continue; // The child page got split, need to reread our page.
-
- default:
- return res;
- }
-
- case NOT_FOUND:
- assert lvl == 0 : lvl;
-
- g.row = null; // Mark not found result.
-
- return res;
-
- default:
- return res;
- }
- }
- }
- finally {
- if (g.canRelease(pageId, lvl))
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param instance Instance name.
- * @param type Tree type.
- * @return Tree name.
- */
- public static String treeName(String instance, String type) {
- return instance + "##" + type;
- }
-
- /**
- * For debug.
- *
- * @return Tree as {@link String}.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unused")
- public final String printTree() throws IgniteCheckedException {
- long rootPageId;
-
- long metaPage = acquirePage(metaPageId);
- try {
- rootPageId = getFirstPageId(metaPageId, metaPage, -1);
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
-
- return treePrinter.print(rootPageId);
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public final void validateTree() throws IgniteCheckedException {
- long rootPageId;
- int rootLvl;
-
- long metaPage = acquirePage(metaPageId);
- try {
- rootLvl = getRootLevel();
-
- if (rootLvl < 0)
- fail("Root level: " + rootLvl);
-
- validateFirstPages(metaPageId, metaPage, rootLvl);
-
- rootPageId = getFirstPageId(metaPageId, metaPage, rootLvl);
-
- validateDownPages(rootPageId, 0L, rootLvl);
-
- validateDownKeys(rootPageId, null, rootLvl);
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
- }
-
- /**
- * @param pageId Page ID.
- * @param minRow Minimum row.
- * @throws IgniteCheckedException If failed.
- */
- private void validateDownKeys(long pageId, L minRow, int lvl) throws IgniteCheckedException {
- long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page); // No correctness guaranties.
-
- try {
- BPlusIO<L> io = io(pageAddr);
-
- int cnt = io.getCount(pageAddr);
-
- if (cnt < 0)
- fail("Negative count: " + cnt);
-
- if (io.isLeaf()) {
- for (int i = 0; i < cnt; i++) {
- if (minRow != null && compare(lvl, io, pageAddr, i, minRow) <= 0)
- fail("Wrong sort order: " + U.hexLong(pageId) + " , at " + i + " , minRow: " + minRow);
-
- minRow = io.getLookupRow(this, pageAddr, i);
- }
-
- return;
- }
-
- // To find our inner key we have to go left and then always go to the right.
- for (int i = 0; i < cnt; i++) {
- L row = io.getLookupRow(this, pageAddr, i);
-
- if (minRow != null && compare(lvl, io, pageAddr, i, minRow) <= 0)
- fail("Min row violated: " + row + " , minRow: " + minRow);
-
- long leftId = inner(io).getLeft(pageAddr, i);
-
- L leafRow = getGreatestRowInSubTree(leftId);
-
- int cmp = compare(lvl, io, pageAddr, i, leafRow);
-
- if (cmp < 0 || (cmp != 0 && canGetRowFromInner))
- fail("Wrong inner row: " + U.hexLong(pageId) + " , at: " + i + " , leaf: " + leafRow +
- " , inner: " + row);
-
- validateDownKeys(leftId, minRow, lvl - 1);
-
- minRow = row;
- }
-
- // Need to handle the rightmost child subtree separately or handle empty routing page.
- long rightId = inner(io).getLeft(pageAddr, cnt); // The same as getRight(cnt - 1)
-
- validateDownKeys(rightId, minRow, lvl - 1);
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param pageId Page ID.
- * @return Search row.
- * @throws IgniteCheckedException If failed.
- */
- private L getGreatestRowInSubTree(long pageId) throws IgniteCheckedException {
- long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page); // No correctness guaranties.
-
- try {
- BPlusIO<L> io = io(pageAddr);
-
- int cnt = io.getCount(pageAddr);
-
- if (io.isLeaf()) {
- if (cnt <= 0) // This code is called only if the tree is not empty, so we can't see empty leaf.
- fail("Invalid leaf count: " + cnt + " " + U.hexLong(pageId));
-
- return io.getLookupRow(this, pageAddr, cnt - 1);
- }
-
- long rightId = inner(io).getLeft(pageAddr, cnt);// The same as getRight(cnt - 1), but good for routing pages.
-
- return getGreatestRowInSubTree(rightId);
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param metaId Meta page ID.
- * @param metaPage Meta page pointer.
- * @param rootLvl Root level.
- * @throws IgniteCheckedException If failed.
- */
- private void validateFirstPages(long metaId, long metaPage, int rootLvl) throws IgniteCheckedException {
- for (int lvl = rootLvl; lvl > 0; lvl--)
- validateFirstPage(metaId, metaPage, lvl);
- }
-
- /**
- * @param msg Message.
- */
- private static void fail(Object msg) {
- throw new AssertionError(msg);
- }
-
- /**
- * @param metaId Meta page ID.
- * @param metaPage Meta page pointer.
- * @param lvl Level.
- * @throws IgniteCheckedException If failed.
- */
- private void validateFirstPage(long metaId, long metaPage, int lvl) throws IgniteCheckedException {
- if (lvl == 0)
- fail("Leaf level: " + lvl);
-
- long pageId = getFirstPageId(metaId, metaPage, lvl);
-
- long leftmostChildId;
-
- long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page); // No correctness guaranties.
-
- try {
- BPlusIO<L> io = io(pageAddr);
-
- if (io.isLeaf())
- fail("Leaf.");
-
- leftmostChildId = inner(io).getLeft(pageAddr, 0);
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
-
- long firstDownPageId = getFirstPageId(metaId, metaPage, lvl - 1);
-
- if (firstDownPageId != leftmostChildId)
- fail(new SB("First: meta ").appendHex(firstDownPageId).a(", child ").appendHex(leftmostChildId));
- }
-
- /**
- * @param pageId Page ID.
- * @param fwdId Forward ID.
- * @param lvl Level.
- * @throws IgniteCheckedException If failed.
- */
- private void validateDownPages(long pageId, long fwdId, int lvl) throws IgniteCheckedException {
- long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page); // No correctness guaranties.
-
- try {
- long realPageId = BPlusIO.getPageId(pageAddr);
-
- if (realPageId != pageId)
- fail(new SB("ABA on page ID: ref ").appendHex(pageId).a(", buf ").appendHex(realPageId));
-
- BPlusIO<L> io = io(pageAddr);
-
- if (io.isLeaf() != (lvl == 0)) // Leaf pages only at the level 0.
- fail("Leaf level mismatch: " + lvl);
-
- long actualFwdId = io.getForward(pageAddr);
-
- if (actualFwdId != fwdId)
- fail(new SB("Triangle: expected fwd ").appendHex(fwdId).a(", actual fwd ").appendHex(actualFwdId));
-
- int cnt = io.getCount(pageAddr);
-
- if (cnt < 0)
- fail("Negative count: " + cnt);
-
- if (io.isLeaf()) {
- if (cnt == 0 && getRootLevel() != 0)
- fail("Empty leaf page.");
- }
- else {
- // Recursively go down if we are on inner level.
- for (int i = 0; i < cnt; i++)
- validateDownPages(inner(io).getLeft(pageAddr, i), inner(io).getRight(pageAddr, i), lvl - 1);
-
- if (fwdId != 0) {
- // For the rightmost child ask neighbor.
- long fwdId0 = fwdId;
- long fwdPage = acquirePage(fwdId0);
- try {
- long fwdPageAddr = readLock(fwdId0, fwdPage); // No correctness guaranties.
-
- try {
- if (io(fwdPageAddr) != io)
- fail("IO on the same level must be the same");
-
- fwdId = inner(io).getLeft(fwdPageAddr, 0);
- }
- finally {
- readUnlock(fwdId0, fwdPage, fwdPageAddr);
- }
- }
- finally {
- releasePage(fwdId0, fwdPage);
- }
- }
-
- long leftId = inner(io).getLeft(pageAddr, cnt); // The same as io.getRight(cnt - 1) but works for routing pages.
-
- validateDownPages(leftId, fwdId, lvl - 1);
- }
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param io IO.
- * @param pageAddr Page address.
- * @param keys Keys.
- * @return String.
- * @throws IgniteCheckedException If failed.
- */
- private String printPage(BPlusIO<L> io, long pageAddr, boolean keys) throws IgniteCheckedException {
- StringBuilder b = new StringBuilder();
-
- b.append(formatPageId(PageIO.getPageId(pageAddr)));
-
- b.append(" [ ");
- b.append(io.isLeaf() ? "L " : "I ");
-
- int cnt = io.getCount(pageAddr);
- long fwdId = io.getForward(pageAddr);
-
- b.append("cnt=").append(cnt).append(' ');
- b.append("fwd=").append(formatPageId(fwdId)).append(' ');
-
- if (!io.isLeaf()) {
- b.append("lm=").append(formatPageId(inner(io).getLeft(pageAddr, 0))).append(' ');
-
- if (cnt > 0)
- b.append("rm=").append(formatPageId(inner(io).getRight(pageAddr, cnt - 1))).append(' ');
- }
-
- if (keys)
- b.append("keys=").append(printPageKeys(io, pageAddr)).append(' ');
-
- b.append(']');
-
- return b.toString();
- }
-
- /**
- * @param io IO.
- * @param pageAddr Page address.
- * @return Keys as String.
- * @throws IgniteCheckedException If failed.
- */
- private String printPageKeys(BPlusIO<L> io, long pageAddr) throws IgniteCheckedException {
- int cnt = io.getCount(pageAddr);
-
- StringBuilder b = new StringBuilder();
-
- b.append('[');
-
- for (int i = 0; i < cnt; i++) {
- if (i != 0)
- b.append(',');
-
- b.append(io.isLeaf() || canGetRowFromInner ? getRow(io, pageAddr, i) : io.getLookupRow(this, pageAddr, i));
- }
-
- b.append(']');
-
- return b.toString();
- }
-
- /**
- * @param x Long.
- * @return String.
- */
- private static String formatPageId(long x) {
- return U.hexLong(x);
- }
-
- /**
- * @param idx Index after binary search, which can be negative.
- * @return Always positive index.
- */
- private static int fix(int idx) {
- assert checkIndex(idx): idx;
-
- if (idx < 0)
- idx = -idx - 1;
-
- return idx;
- }
-
- /**
- * @param idx Index.
- * @return {@code true} If correct.
- */
- private static boolean checkIndex(int idx) {
- return idx > -Short.MAX_VALUE && idx < Short.MAX_VALUE;
- }
-
- /**
- * Check if interrupted.
- * @throws IgniteInterruptedCheckedException If interrupted.
- */
- private static void checkInterrupted() throws IgniteInterruptedCheckedException {
- // We should not interrupt operations in the middle, because otherwise we'll end up in inconsistent state.
- // Because of that we do not check for Thread.interrupted()
- if (interrupted)
- throw new IgniteInterruptedCheckedException("Interrupted.");
- }
-
- /**
- * Interrupt all operations on all threads and all indexes.
- */
- @SuppressWarnings("unused")
- public static void interruptAll() {
- interrupted = true;
- }
-
- /**
- * @param row Lookup row.
- * @return Removed row.
- * @throws IgniteCheckedException If failed.
- */
- @Override public final T remove(L row) throws IgniteCheckedException {
- return doRemove(row, true);
- }
-
- /**
- * @param row Lookup row.
- * @throws IgniteCheckedException If failed.
- * @return {@code True} if removed row.
- */
- public final boolean removex(L row) throws IgniteCheckedException {
- Boolean res = (Boolean)doRemove(row, false);
-
- return res != null ? res : false;
- }
-
- /** {@inheritDoc} */
- @Override public void invoke(L row, Object z, InvokeClosure<T> c) throws IgniteCheckedException {
- checkDestroyed();
-
- Invoke x = new Invoke(row, z, c);
-
- try {
- for (;;) {
- x.init();
-
- Result res = invokeDown(x, x.rootId, 0L, 0L, x.rootLvl);
-
- switch (res) {
- case RETRY:
- case RETRY_ROOT:
- checkInterrupted();
-
- continue;
-
- default:
- if (!x.isFinished()) {
- res = x.tryFinish();
-
- if (res == RETRY || res == RETRY_ROOT) {
- checkInterrupted();
-
- continue;
- }
-
- assert x.isFinished(): res;
- }
-
- return;
- }
- }
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on search row: " + row, e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on search row: " + row, e);
- }
- finally {
- x.releaseAll();
- checkDestroyed();
- }
- }
-
- /**
- * @param x Invoke operation.
- * @param pageId Page ID.
- * @param backId Expected backward page ID if we are going to the right.
- * @param fwdId Expected forward page ID.
- * @param lvl Level.
- * @return Result code.
- * @throws IgniteCheckedException If failed.
- */
- private Result invokeDown(final Invoke x, final long pageId, final long backId, final long fwdId, final int lvl)
- throws IgniteCheckedException {
- assert lvl >= 0 : lvl;
-
- if (x.isTail(pageId, lvl))
- return FOUND; // We've already locked this page, so return that we are ok.
-
- long page = acquirePage(pageId);
-
- try {
- for (;;) {
- // Init args.
- x.pageId(pageId);
- x.fwdId(fwdId);
- x.backId(backId);
-
- Result res = read(pageId, page, search, x, lvl, RETRY);
-
- switch (res) {
- case GO_DOWN_X:
- assert backId != 0;
- assert x.backId == 0; // We did not setup it yet.
-
- x.backId(pageId); // Dirty hack to setup a check inside of askNeighbor.
-
- // We need to get backId here for our child page, it must be the last child of our back.
- res = askNeighbor(backId, x, true);
-
- if (res != FOUND)
- return res; // Retry.
-
- assert x.backId != pageId; // It must be updated in askNeighbor.
-
- // Intentional fallthrough.
- case GO_DOWN:
- res = x.tryReplaceInner(pageId, page, fwdId, lvl);
-
- if (res != RETRY)
- res = invokeDown(x, x.pageId, x.backId, x.fwdId, lvl - 1);
-
- if (res == RETRY_ROOT || x.isFinished())
- return res;
-
- if (res == RETRY) {
- checkInterrupted();
-
- continue;
- }
-
- // Unfinished Put does insertion on the same level.
- if (x.isPut())
- continue;
-
- assert x.isRemove(); // Guarded by isFinished.
-
- res = x.finishOrLockTail(pageId, page, backId, fwdId, lvl);
-
- return res;
-
- case NOT_FOUND:
- if (lvl == 0)
- x.invokeClosure();
-
- return x.onNotFound(pageId, page, fwdId, lvl);
-
- case FOUND:
- if (lvl == 0)
- x.invokeClosure();
-
- return x.onFound(pageId, page, backId, fwdId, lvl);
-
- default:
- return res;
- }
- }
- }
- finally {
- x.levelExit();
-
- if (x.canRelease(pageId, lvl))
- releasePage(pageId, page);
- }
- }
-
-
- /**
- * @param row Lookup row.
- * @param needOld {@code True} if need return removed row.
- * @return Removed row.
- * @throws IgniteCheckedException If failed.
- */
- private T doRemove(L row, boolean needOld) throws IgniteCheckedException {
- checkDestroyed();
-
- Remove r = new Remove(row, needOld);
-
- try {
- for (;;) {
- r.init();
-
- Result res = removeDown(r, r.rootId, 0L, 0L, r.rootLvl);
-
- switch (res) {
- case RETRY:
- case RETRY_ROOT:
- checkInterrupted();
-
- continue;
-
- default:
- if (!r.isFinished()) {
- res = r.finishTail();
-
- // If not found, then the tree grew beyond our call stack -> retry from the actual root.
- if (res == RETRY || res == NOT_FOUND) {
- assert r.checkTailLevel(getRootLevel()) : "tail=" + r.tail + ", res=" + res;
-
- checkInterrupted();
-
- continue;
- }
-
- assert res == FOUND: res;
- }
-
- assert r.isFinished();
-
- return r.rmvd;
- }
- }
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on search row: " + row, e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on search row: " + row, e);
- }
- finally {
- r.releaseAll();
- checkDestroyed();
- }
- }
-
- /**
- * @param r Remove operation.
- * @param pageId Page ID.
- * @param backId Expected backward page ID if we are going to the right.
- * @param fwdId Expected forward page ID.
- * @param lvl Level.
- * @return Result code.
- * @throws IgniteCheckedException If failed.
- */
- private Result removeDown(final Remove r, final long pageId, final long backId, final long fwdId, final int lvl)
- throws IgniteCheckedException {
- assert lvl >= 0 : lvl;
-
- if (r.isTail(pageId, lvl))
- return FOUND; // We've already locked this page, so return that we are ok.
-
- long page = acquirePage(pageId);
-
- try {
- for (;;) {
- // Init args.
- r.pageId = pageId;
- r.fwdId = fwdId;
- r.backId = backId;
-
- Result res = read(pageId, page, search, r, lvl, RETRY);
-
- switch (res) {
- case GO_DOWN_X:
- assert backId != 0;
- assert r.backId == 0; // We did not setup it yet.
-
- r.backId = pageId; // Dirty hack to setup a check inside of askNeighbor.
-
- // We need to get backId here for our child page, it must be the last child of our back.
- res = askNeighbor(backId, r, true);
-
- if (res != FOUND)
- return res; // Retry.
-
- assert r.backId != pageId; // It must be updated in askNeighbor.
-
- // Intentional fallthrough.
- case GO_DOWN:
- res = removeDown(r, r.pageId, r.backId, r.fwdId, lvl - 1);
-
- if (res == RETRY) {
- checkInterrupted();
-
- continue;
- }
-
- if (res == RETRY_ROOT || r.isFinished())
- return res;
-
- res = r.finishOrLockTail(pageId, page, backId, fwdId, lvl);
-
- return res;
-
- case NOT_FOUND:
- // We are at the bottom.
- assert lvl == 0 : lvl;
-
- r.finish();
-
- return res;
-
- case FOUND:
- return r.tryRemoveFromLeaf(pageId, page, backId, fwdId, lvl);
-
- default:
- return res;
- }
- }
- }
- finally {
- r.page = 0L;
-
- if (r.canRelease(pageId, lvl))
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param cnt Count.
- * @param cap Capacity.
- * @return {@code true} If may merge.
- */
- private boolean mayMerge(int cnt, int cap) {
- int minCnt = (int)(minFill * cap);
-
- if (cnt <= minCnt) {
- assert cnt == 0; // TODO remove
-
- return true;
- }
-
- assert cnt > 0;
-
- int maxCnt = (int)(maxFill * cap);
-
- if (cnt > maxCnt)
- return false;
-
- assert false; // TODO remove
-
- // Randomization is for smoothing worst case scenarios. Probability of merge attempt
- // is proportional to free space in our page (discounted on fill factor).
- return randomInt(maxCnt - minCnt) >= cnt - minCnt;
- }
-
- /**
- * @return Root level.
- * @throws IgniteCheckedException If failed.
- */
- public final int rootLevel() throws IgniteCheckedException {
- checkDestroyed();
-
- return getRootLevel();
- }
-
- /**
- * !!! For debug only! May produce wrong results on concurrent access.
- *
- * @return Size.
- * @throws IgniteCheckedException If failed.
- */
- @Override public final long size() throws IgniteCheckedException {
- checkDestroyed();
-
- long pageId;
-
- long metaPage = acquirePage(metaPageId);
- try {
- pageId = getFirstPageId(metaPageId, metaPage, 0); // Level 0 is always at the bottom.
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
-
- BPlusIO<L> io = null;
-
- long cnt = 0;
-
- while (pageId != 0) {
- long curId = pageId;
- long curPage = acquirePage(curId);
- try {
- long curAddr = readLock(curId, curPage); // No correctness guaranties.
-
- try {
- if (io == null) {
- io = io(curAddr);
-
- assert io.isLeaf();
- }
-
- cnt += io.getCount(curAddr);
-
- pageId = io.getForward(curAddr);
- }
- finally {
- readUnlock(curId, curPage, curAddr);
- }
- }
- finally {
- releasePage(curId, curPage);
- }
- }
-
- checkDestroyed();
-
- return cnt;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public final T put(T row) throws IgniteCheckedException {
- return doPut(row, true);
- }
-
- /**
- * @param row New value.
- * @throws IgniteCheckedException If failed.
- * @return {@code True} if replaced existing row.
- */
- public boolean putx(T row) throws IgniteCheckedException {
- Boolean res = (Boolean)doPut(row, false);
-
- return res != null ? res : false;
- }
-
- /**
- * @param row New value.
- * @param needOld {@code True} If need return old value.
- * @return Old row.
- * @throws IgniteCheckedException If failed.
- */
- private T doPut(T row, boolean needOld) throws IgniteCheckedException {
- checkDestroyed();
-
- Put p = new Put(row, needOld);
-
- try {
- for (;;) { // Go down with retries.
- p.init();
-
- Result res = putDown(p, p.rootId, 0L, p.rootLvl);
-
- switch (res) {
- case RETRY:
- case RETRY_ROOT:
- checkInterrupted();
-
- continue;
-
- case FOUND:
- // We may need to insert split key into upper level here.
- if (!p.isFinished()) {
- // It must be impossible to have an insert higher than the current root,
- // because we are making decision about creating new root while keeping
- // write lock on current root, so it can't concurrently change.
- assert p.btmLvl <= getRootLevel();
-
- checkInterrupted();
-
- continue;
- }
-
- return p.oldRow;
-
- default:
- throw new IllegalStateException("Result: " + res);
- }
- }
- }
- catch (IgniteCheckedException e) {
- throw new IgniteCheckedException("Runtime failure on row: " + row, e);
- }
- catch (RuntimeException e) {
- throw new IgniteException("Runtime failure on row: " + row, e);
- }
- catch (AssertionError e) {
- throw new AssertionError("Assertion error on row: " + row, e);
- }
- finally {
- checkDestroyed();
- }
- }
-
- /**
- * Destroys tree. This method is allowed to be invoked only when the tree is out of use (no concurrent operations
- * are trying to read or update the tree after destroy beginning).
- *
- * @return Number of pages recycled from this tree. If the tree was destroyed by someone else concurrently returns
- * {@code 0}, otherwise it should return at least {@code 2} (for meta page and root page), unless this tree is
- * used as metadata storage, or {@code -1} if we don't have a reuse list and did not do recycling at all.
- * @throws IgniteCheckedException If failed.
- */
- public final long destroy() throws IgniteCheckedException {
- return destroy(null);
- }
-
- /**
- * Destroys tree. This method is allowed to be invoked only when the tree is out of use (no concurrent operations
- * are trying to read or update the tree after destroy beginning).
- *
- * @param c Visitor closure. Visits only leaf pages.
- * @return Number of pages recycled from this tree. If the tree was destroyed by someone else concurrently returns
- * {@code 0}, otherwise it should return at least {@code 2} (for meta page and root page), unless this tree is
- * used as metadata storage, or {@code -1} if we don't have a reuse list and did not do recycling at all.
- * @throws IgniteCheckedException If failed.
- */
- public final long destroy(IgniteInClosure<L> c) throws IgniteCheckedException {
- if (!markDestroyed())
- return 0;
-
- if (reuseList == null)
- return -1;
-
- DestroyBag bag = new DestroyBag();
-
- long pagesCnt = 0;
-
- long metaPage = acquirePage(metaPageId);
- try {
- long metaPageAddr = writeLock(metaPageId, metaPage); // No checks, we must be out of use.
-
- assert metaPageAddr != 0L;
-
- try {
- for (long pageId : getFirstPageIds(metaPageAddr)) {
- assert pageId != 0;
-
- do {
- final long pId = pageId;
-
- long page = acquirePage(pId);
-
- try {
- long pageAddr = writeLock(pId, page); // No checks, we must be out of use.
-
- try {
- BPlusIO<L> io = io(pageAddr);
-
- if (c != null && io.isLeaf())
- io.visit(pageAddr, c);
-
- long fwdPageId = io.getForward(pageAddr);
-
- bag.addFreePage(recyclePage(pageId, page, pageAddr, null));
- pagesCnt++;
-
- pageId = fwdPageId;
- }
- finally {
- writeUnlock(pId, page, pageAddr, true);
- }
- }
- finally {
- releasePage(pId, page);
- }
-
- if (bag.size() == 128) {
- reuseList.addForRecycle(bag);
-
- assert bag.isEmpty() : bag.size();
- }
- }
- while (pageId != 0);
- }
-
- bag.addFreePage(recyclePage(metaPageId, metaPage, metaPageAddr, null));
- pagesCnt++;
- }
- finally {
- writeUnlock(metaPageId, metaPage, metaPageAddr, true);
- }
- }
- finally {
- releasePage(metaPageId, metaPage);
- }
-
- reuseList.addForRecycle(bag);
-
- assert bag.size() == 0 : bag.size();
-
- return pagesCnt;
- }
-
- /**
- * @return {@code True} if state was changed.
- */
- private boolean markDestroyed() {
- return destroyed.compareAndSet(false, true);
- }
-
- /**
- * @param pageAddr Meta page address.
- * @return First page IDs.
- */
- protected Iterable<Long> getFirstPageIds(long pageAddr) {
- List<Long> res = new ArrayList<>();
-
- BPlusMetaIO mio = BPlusMetaIO.VERSIONS.forPage(pageAddr);
-
- for (int lvl = mio.getRootLevel(pageAddr); lvl >= 0; lvl--)
- res.add(mio.getFirstPageId(pageAddr, lvl));
-
- return res;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address
- * @param io IO.
- * @param fwdId Forward page ID.
- * @param fwdBuf Forward buffer.
- * @param idx Insertion index.
- * @return {@code true} The middle index was shifted to the right.
- * @throws IgniteCheckedException If failed.
- */
- private boolean splitPage(
- long pageId, long page, long pageAddr, BPlusIO io, long fwdId, long fwdBuf, int idx
- ) throws IgniteCheckedException {
- int cnt = io.getCount(pageAddr);
- int mid = cnt >>> 1;
-
- boolean res = false;
-
- if (idx > mid) { // If insertion is going to be to the forward page, keep more in the back page.
- mid++;
-
- res = true;
- }
-
- // Update forward page.
- io.splitForwardPage(pageAddr, fwdId, fwdBuf, mid, cnt, pageSize());
-
- // Update existing page.
- io.splitExistingPage(pageAddr, mid, fwdId);
-
- if (needWalDeltaRecord(pageId, page, null))
- wal.log(new SplitExistingPageRecord(cacheId, pageId, mid, fwdId));
-
- return res;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address
- * @param walPlc Full page WAL record policy.
- */
- private void writeUnlockAndClose(long pageId, long page, long pageAddr, Boolean walPlc) {
- try {
- writeUnlock(pageId, page, pageAddr, walPlc, true);
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param pageId Inner page ID.
- * @param g Get.
- * @param back Get back (if {@code true}) or forward page (if {@code false}).
- * @return Operation result.
- * @throws IgniteCheckedException If failed.
- */
- private Result askNeighbor(long pageId, Get g, boolean back) throws IgniteCheckedException {
- return read(pageId, askNeighbor, g, back ? TRUE.ordinal() : FALSE.ordinal(), RETRY);
- }
-
- /**
- * @param p Put.
- * @param pageId Page ID.
- * @param fwdId Expected forward page ID.
- * @param lvl Level.
- * @return Result code.
- * @throws IgniteCheckedException If failed.
- */
- private Result putDown(final Put p, final long pageId, final long fwdId, final int lvl)
- throws IgniteCheckedException {
- assert lvl >= 0 : lvl;
-
- final long page = acquirePage(pageId);
-
- try {
- for (;;) {
- // Init args.
- p.pageId = pageId;
- p.fwdId = fwdId;
-
- Result res = read(pageId, page, search, p, lvl, RETRY);
-
- switch (res) {
- case GO_DOWN:
- case GO_DOWN_X:
- assert lvl > 0 : lvl;
- assert p.pageId != pageId;
- assert p.fwdId != fwdId || fwdId == 0;
-
- res = p.tryReplaceInner(pageId, page, fwdId, lvl);
-
- if (res != RETRY) // Go down recursively.
- res = putDown(p, p.pageId, p.fwdId, lvl - 1);
-
- if (res == RETRY_ROOT || p.isFinished())
- return res;
-
- if (res == RETRY)
- checkInterrupted();
-
- continue; // We have to insert split row to this level or it is a retry.
-
- case FOUND: // Do replace.
- assert lvl == 0 : "This replace can happen only at the bottom level.";
-
- return p.tryReplace(pageId, page, fwdId, lvl);
-
- case NOT_FOUND: // Do insert.
- assert lvl == p.btmLvl : "must insert at the bottom level";
- assert p.needReplaceInner == FALSE : p.needReplaceInner + " " + lvl;
-
- return p.tryInsert(pageId, page, fwdId, lvl);
-
- default:
- return res;
- }
- }
- }
- finally {
- if (p.canRelease(pageId, lvl))
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param io IO.
- * @param pageAddr Page address.
- * @param back Backward page.
- * @return Page ID.
- */
- private long doAskNeighbor(BPlusIO<L> io, long pageAddr, boolean back) {
- long res;
-
- if (back) {
- // Count can be 0 here if it is a routing page, in this case we have a single child.
- int cnt = io.getCount(pageAddr);
-
- // We need to do get the rightmost child: io.getRight(cnt - 1),
- // here io.getLeft(cnt) is the same, but handles negative index if count is 0.
- res = inner(io).getLeft(pageAddr, cnt);
- }
- else // Leftmost child.
- res = inner(io).getLeft(pageAddr, 0);
-
- assert res != 0 : "inner page with no route down: " + U.hexLong(PageIO.getPageId(pageAddr));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(BPlusTree.class, this);
- }
-
- /**
- * Get operation.
- */
- private abstract class Get {
- /** */
- long rmvId;
-
- /** Starting point root level. May be outdated. Must be modified only in {@link Get#init()}. */
- int rootLvl;
-
- /** Starting point root ID. May be outdated. Must be modified only in {@link Get#init()}. */
- long rootId;
-
- /** */
- L row;
-
- /** In/Out parameter: Page ID. */
- long pageId;
-
- /** In/Out parameter: expected forward page ID. */
- long fwdId;
-
- /** In/Out parameter: in case of right turn this field will contain backward page ID for the child. */
- long backId;
-
- /** */
- int shift;
-
- /** If this operation is a part of invoke. */
- Invoke invoke;
-
- /** Ignore row passed, find last row */
- boolean findLast;
-
- /**
- * @param row Row.
- * @param findLast find last row.
- */
- Get(L row, boolean findLast) {
- assert findLast ^ row != null;
-
- this.row = row;
- this.findLast = findLast;
- }
-
- /**
- * @param g Other operation to copy from.
- */
- final void copyFrom(Get g) {
- rmvId = g.rmvId;
- rootLvl = g.rootLvl;
- pageId = g.pageId;
- fwdId = g.fwdId;
- backId = g.backId;
- shift = g.shift;
- findLast = g.findLast;
- }
-
- /**
- * Initialize operation.
- *
- * @throws IgniteCheckedException If failed.
- */
- final void init() throws IgniteCheckedException {
- TreeMetaData meta0 = treeMeta();
-
- assert meta0 != null;
-
- restartFromRoot(meta0.rootId, meta0.rootLvl, globalRmvId.get());
- }
-
- /**
- * @param rootId Root page ID.
- * @param rootLvl Root level.
- * @param rmvId Remove ID to be afraid of.
- */
- void restartFromRoot(long rootId, int rootLvl, long rmvId) {
- this.rootId = rootId;
- this.rootLvl = rootLvl;
- this.rmvId = rmvId;
- }
-
- /**
- * @param io IO.
- * @param pageAddr Page address.
- * @param idx Index of found entry.
- * @param lvl Level.
- * @return {@code true} If we need to stop.
- * @throws IgniteCheckedException If failed.
- */
- boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
- assert lvl >= 0;
-
- return lvl == 0; // Stop if we are at the bottom.
- }
-
- /**
- * @param io IO.
- * @param pageAddr Page address.
- * @param idx Insertion point.
- * @param lvl Level.
- * @return {@code true} If we need to stop.
- * @throws IgniteCheckedException If failed.
- */
- boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
- assert lvl >= 0;
-
- return lvl == 0; // Stop if we are at the bottom.
- }
-
- /**
- * @param pageId Page.
- * @param lvl Level.
- * @return {@code true} If we can release the given page.
- */
- boolean canRelease(long pageId, int lvl) {
- return pageId != 0L;
- }
-
- /**
- * @param backId Back page ID.
- */
- void backId(long backId) {
- this.backId = backId;
- }
-
- /**
- * @param pageId Page ID.
- */
- void pageId(long pageId) {
- this.pageId = pageId;
- }
-
- /**
- * @param fwdId Forward page ID.
- */
- void fwdId(long fwdId) {
- this.fwdId = fwdId;
- }
-
- /**
- * @return {@code true} If the operation is finished.
- */
- boolean isFinished() {
- throw new IllegalStateException();
- }
- }
-
- /**
- * Get a single entry.
- */
- private final class GetOne extends Get {
- /** */
- Object x;
-
- /**
- * @param row Row.
- * @param x Implementation specific argument.
- * @param findLast Ignore row passed, find last row
- */
- private GetOne(L row, Object x, boolean findLast) {
- super(row, findLast);
-
- this.x = x;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("unchecked")
- @Override boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
- // Check if we are on an inner page and can't get row from it.
- if (lvl != 0 && !canGetRowFromInner)
- return false;
-
- row = getRow(io, pageAddr, idx, x);
-
- return true;
- }
- }
-
- /**
- * Get a cursor for range.
- */
- private final class GetCursor extends Get {
- /** */
- ForwardCursor cursor;
-
- /**
- * @param lower Lower bound.
- * @param shift Shift.
- * @param cursor Cursor.
- */
- GetCursor(L lower, int shift, ForwardCursor cursor) {
- super(lower, false);
-
- assert shift != 0; // Either handle range of equal rows or find a greater row after concurrent merge.
-
- this.shift = shift;
- this.cursor = cursor;
- }
-
- /** {@inheritDoc} */
- @Override boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
- throw new IllegalStateException(); // Must never be called because we always have a shift.
- }
-
- /** {@inheritDoc} */
- @Override boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
- if (lvl != 0)
- return false;
-
- cursor.init(pageAddr, io, idx);
-
- return true;
- }
- }
-
- /**
- * Put operation.
- */
- private final class Put extends Get {
- /** Right child page ID for split row. */
- long rightId;
-
- /** Replaced row if any. */
- T oldRow;
-
- /**
- * This page is kept locked after split until insert to the upper level will not be finished.
- * It is needed because split row will be "in flight" and if we'll release tail, remove on
- * split row may fail.
- */
- long tailId;
-
- /** */
- long tailPage;
-
- /** */
- long tailAddr;
-
- /**
- * Bottom level for insertion (insert can't go deeper). Will be incremented on split on each level.
- */
- short btmLvl;
-
- /** */
- Bool needReplaceInner = FALSE;
-
- /** */
- final boolean needOld;
-
- /**
- * @param row Row.
- * @param needOld {@code True} If need return old value.
- */
- private Put(T row, boolean needOld) {
- super(row, false);
-
- this.needOld = needOld;
- }
-
- /** {@inheritDoc} */
- @Override boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) {
- if (lvl == 0) // Leaf: need to stop.
- return true;
-
- assert btmLvl == 0; // It can not be insert.
-
- // If we can get full row from the inner page, we have to replace it with the new one. On the way down
- // we can not miss inner key even in presence of concurrent operations because of `triangle` invariant +
- // concurrent inner replace handling by retrying from root.
- if (canGetRowFromInner && needReplaceInner == FALSE)
- needReplaceInner = TRUE;
-
- return false;
- }
-
- /** {@inheritDoc} */
- @Override boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) {
- assert btmLvl >= 0 : btmLvl;
- assert lvl >= btmLvl : lvl;
-
- return lvl == btmLvl;
- }
-
- /**
- * @param tailId Tail page ID.
- * @param tailPage Tail page pointer.
- * @param tailPageAddr Tail page address
- */
- private void tail(long tailId, long tailPage, long tailPageAddr) {
- assert (tailId == 0L) == (tailPage == 0L);
- assert (tailPage == 0L) == (tailPageAddr == 0L);
-
- if (this.tailPage != 0L)
- writeUnlockAndClose(this.tailId, this.tailPage, this.tailAddr, null);
-
- this.tailId = tailId;
- this.tailPage = tailPage;
- this.tailAddr = tailPageAddr;
- }
-
- /** {@inheritDoc} */
- @Override boolean canRelease(long pageId, int lvl) {
- return pageId != 0L && tailId != pageId;
- }
-
- /**
- * Finish put.
- */
- private void finish() {
- row = null;
- rightId = 0;
-
- tail(0L, 0L, 0L);
- }
-
- /** {@inheritDoc} */
- @Override boolean isFinished() {
- return row == null;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param idx Index.
- * @param lvl Level.
- * @return Move up row.
- * @throws IgniteCheckedException If failed.
- */
- private L insert(long pageId, long page, long pageAddr, BPlusIO<L> io, int idx, int lvl)
- throws IgniteCheckedException {
- int maxCnt = io.getMaxCount(pageAddr, pageSize());
-
<TRUNCATED>
[41/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/IOVersions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/IOVersions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/IOVersions.java
deleted file mode 100644
index 3545b32..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/IOVersions.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-/**
- * Registry for IO versions.
- */
-public final class IOVersions<V extends PageIO> {
- /** */
- private final V[] vers;
-
- /** */
- private final int type;
-
- /** */
- private final V latest;
-
- /**
- * @param vers Versions.
- */
- @SafeVarargs
- public IOVersions(V... vers) {
- assert vers != null;
- assert vers.length > 0;
-
- this.vers = vers;
- this.type = vers[0].getType();
-
- latest = vers[vers.length - 1];
-
- assert checkVersions();
- }
-
- /**
- * @return Type.
- */
- public int getType() {
- return type;
- }
-
- /**
- * @return {@code true} If versions are correct.
- */
- private boolean checkVersions() {
- for (int i = 0; i < vers.length; i++) {
- V v = vers[i];
-
- if (v.getType() != type || v.getVersion() != i + 1)
- return false;
- }
-
- return true;
- }
-
- /**
- * @return Latest IO version.
- */
- public V latest() {
- return latest;
- }
-
- /**
- * @param ver Version.
- * @return IO.
- */
- public V forVersion(int ver) {
- return vers[ver - 1];
- }
-
- /**
- * @param pageAddr Page address.
- * @return IO.
- */
- public V forPage(long pageAddr) {
- int ver = PageIO.getVersion(pageAddr);
-
- V res = forVersion(ver);
-
- assert res.getType() == PageIO.getType(pageAddr) : "resType=" + res.getType() +
- ", pageType=" + PageIO.getType(pageAddr);
-
- return res;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
deleted file mode 100644
index dd6a503..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageIO.java
+++ /dev/null
@@ -1,548 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl;
-import org.apache.ignite.internal.processors.cache.database.MetadataStorage;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListMetaIO;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageLockListener;
-
-/**
- * Base format for all the page types.
- *
- * Checklist for page IO implementations and usage (The Rules):
- *
- * 1. IO should not have any `public static` methods.
- * We have versioned IOs and any static method will mean that it have to always work in backward
- * compatible way between all the IO versions. The base class {@link PageIO} has
- * static methods (like {@code {@link #getPageId(long)}}) intentionally:
- * this base format can not be changed between versions.
- *
- * 2. IO must correctly override {@link #initNewPage(long, long, int)} method and call super.
- * We have logic that relies on this behavior.
- *
- * 3. Page IO type ID constant must be declared in this class to have a list of all the
- * existing IO types in a single place.
- *
- * 4. IO must be added to {@link #getBPlusIO(int, int)} or to {@link #getPageIO(int, int)}.
- *
- * 5. Always keep in mind that IOs are versioned and their format can change from version
- * to version. In this respect it is a good practice to avoid exposing details
- * of IO internal format on it's API. The API should be minimalistic and abstract, so that
- * internal format in future IO version can be completely changed without any changes
- * to the API of this page IO.
- *
- * 6. Page IO API should not have any version dependent semantics and should not change API
- * semantics in newer versions.
- *
- * 7. It is almost always preferable to read or write (especially write) page contents using
- * static methods on {@link PageHandler}. To just initialize new page use
- * {@link PageHandler#initPage(PageMemory, int, long, PageIO, IgniteWriteAheadLogManager, PageLockListener)}
- * method with needed IO instance.
- */
-public abstract class PageIO {
- /** */
- private static BPlusInnerIO<?> innerTestIO;
-
- /** */
- private static BPlusLeafIO<?> leafTestIO;
-
- /** */
- private static IOVersions<? extends BPlusInnerIO<?>> h2InnerIOs;
-
- /** */
- private static IOVersions<? extends BPlusLeafIO<?>> h2LeafIOs;
-
- /** Maximum payload size. */
- public static final short MAX_PAYLOAD_SIZE = 2048;
-
- /** */
- private static List<IOVersions<? extends BPlusInnerIO<?>>> h2ExtraInnerIOs = new ArrayList<>(MAX_PAYLOAD_SIZE);
-
- /** */
- private static List<IOVersions<? extends BPlusLeafIO<?>>> h2ExtraLeafIOs = new ArrayList<>(MAX_PAYLOAD_SIZE);
-
- /** */
- public static final int TYPE_OFF = 0;
-
- /** */
- public static final int VER_OFF = TYPE_OFF + 2;
-
- /** */
- public static final int CRC_OFF = VER_OFF + 2;
-
- /** */
- public static final int PAGE_ID_OFF = CRC_OFF + 4;
-
- /** */
- private static final int RESERVED_1_OFF = PAGE_ID_OFF + 8;
-
- /** */
- private static final int RESERVED_2_OFF = RESERVED_1_OFF + 8;
-
- /** */
- private static final int RESERVED_3_OFF = RESERVED_2_OFF + 8;
-
- /** */
- public static final int COMMON_HEADER_END = RESERVED_3_OFF + 8; // 40=type(2)+ver(2)+crc(4)+pageId(8)+reserved(3*8)
-
- /* All the page types. */
-
- /** */
- public static final short T_DATA = 1;
-
- /** */
- public static final short T_BPLUS_META = 2;
-
- /** */
- public static final short T_H2_REF_LEAF = 3;
-
- /** */
- public static final short T_H2_REF_INNER = 4;
-
- /** */
- public static final short T_DATA_REF_INNER = 5;
-
- /** */
- public static final short T_DATA_REF_LEAF = 6;
-
- /** */
- public static final short T_METASTORE_INNER = 7;
-
- /** */
- public static final short T_METASTORE_LEAF = 8;
-
- /** */
- public static final short T_PENDING_REF_INNER = 9;
-
- /** */
- public static final short T_PENDING_REF_LEAF = 10;
-
- /** */
- public static final short T_META = 11;
-
- /** */
- public static final short T_PAGE_LIST_META = 12;
-
- /** */
- public static final short T_PAGE_LIST_NODE = 13;
-
- /** */
- public static final short T_PART_META = 14;
-
- /** */
- public static final short T_PAGE_UPDATE_TRACKING = 15;
-
- /** */
- public static final short T_CACHE_ID_AWARE_DATA_REF_INNER = 16;
-
- /** */
- public static final short T_CACHE_ID_AWARE_DATA_REF_LEAF = 17;
-
- /** */
- public static final short T_CACHE_ID_AWARE_PENDING_REF_INNER = 18;
-
- /** */
- public static final short T_CACHE_ID_AWARE_PENDING_REF_LEAF = 19;
-
- /** */
- public static final short T_PART_CNTRS = 20;
-
- /** Index for payload == 1. */
- public static final short T_H2_EX_REF_LEAF_START = 10000;
-
- /** */
- public static final short T_H2_EX_REF_LEAF_END = T_H2_EX_REF_LEAF_START + MAX_PAYLOAD_SIZE - 1;
-
- /** */
- public static final short T_H2_EX_REF_INNER_START = 20000;
-
- /** */
- public static final short T_H2_EX_REF_INNER_END = T_H2_EX_REF_INNER_START + MAX_PAYLOAD_SIZE - 1;
-
- /** */
- private final int ver;
-
- /** */
- private final int type;
-
- /**
- * @param type Page type.
- * @param ver Page format version.
- */
- protected PageIO(int type, int ver) {
- assert ver > 0 && ver < 65535: ver;
- assert type > 0 && type < 65535: type;
-
- this.type = type;
- this.ver = ver;
- }
-
- /**
- * @param buf Buffer.
- * @return Page type.
- */
- public static int getType(ByteBuffer buf) {
- return buf.getShort(TYPE_OFF) & 0xFFFF;
- }
-
- /**
- * @param pageAddr Page addres.
- * @return Page type.
- */
- public static int getType(long pageAddr) {
- return PageUtils.getShort(pageAddr, TYPE_OFF) & 0xFFFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @param type Type.
- */
- public static void setType(long pageAddr, int type) {
- PageUtils.putShort(pageAddr, TYPE_OFF, (short)type);
-
- assert getType(pageAddr) == type;
- }
-
- /**
- * @param buf Buffer.
- * @return Version.
- */
- public static int getVersion(ByteBuffer buf) {
- return buf.getShort(VER_OFF) & 0xFFFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Version.
- */
- public static int getVersion(long pageAddr) {
- return PageUtils.getShort(pageAddr, VER_OFF) & 0xFFFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @param ver Version.
- */
- private static void setVersion(long pageAddr, int ver) {
- PageUtils.putShort(pageAddr, VER_OFF, (short)ver);
-
- assert getVersion(pageAddr) == ver;
- }
-
- /**
- * @param buf Buffer.
- * @return Page ID.
- */
- public static long getPageId(ByteBuffer buf) {
- return buf.getLong(PAGE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Page ID.
- */
- public static long getPageId(long pageAddr) {
- return PageUtils.getLong(pageAddr, PAGE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageId Page ID.
- */
- public static void setPageId(long pageAddr, long pageId) {
- PageUtils.putLong(pageAddr, PAGE_ID_OFF, pageId);
-
- assert getPageId(pageAddr) == pageId;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Checksum.
- */
- public static int getCrc(long pageAddr) {
- return PageUtils.getInt(pageAddr, CRC_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param crc Checksum.
- */
- public static void setCrc(long pageAddr, int crc) {
- PageUtils.putInt(pageAddr, CRC_OFF, crc);
- }
-
- /**
- * @param buf Buffer.
- * @return Checksum.
- */
- public static int getCrc(ByteBuffer buf) {
- return buf.getInt(CRC_OFF);
- }
-
- /**
- * @param buf Buffer.
- * @param crc Checksum.
- */
- public static void setCrc(ByteBuffer buf, int crc) {
- buf.putInt(CRC_OFF, crc);
- }
-
- /**
- * Registers this B+Tree IO versions.
- *
- * @param innerIOs Inner IO versions.
- * @param leafIOs Leaf IO versions.
- */
- public static void registerH2(
- IOVersions<? extends BPlusInnerIO<?>> innerIOs,
- IOVersions<? extends BPlusLeafIO<?>> leafIOs
- ) {
- h2InnerIOs = innerIOs;
- h2LeafIOs = leafIOs;
- }
-
- /**
- * Registers extra inner IO versions.
- *
- * @param innerExtIOs Extra versions.
- */
- public static void registerH2ExtraInner(IOVersions<? extends BPlusInnerIO<?>> innerExtIOs) {
- h2ExtraInnerIOs.add(innerExtIOs);
- }
-
- /**
- * Registers extra inner IO versions.
- *
- * @param leafExtIOs Extra versions.
- */
- public static void registerH2ExtraLeaf(IOVersions<? extends BPlusLeafIO<?>> leafExtIOs) {
- h2ExtraLeafIOs.add(leafExtIOs);
- }
-
- /**
- * @param idx Index.
- * @return IOVersions for given idx.
- */
- public static IOVersions<? extends BPlusInnerIO<?>> getInnerVersions(int idx) {
- return h2ExtraInnerIOs.get(idx);
- }
-
- /**
- * @param idx Index.
- * @return IOVersions for given idx.
- */
- public static IOVersions<? extends BPlusLeafIO<?>> getLeafVersions(int idx) {
- return h2ExtraLeafIOs.get(idx);
- }
-
- /**
- * Registers IOs for testing.
- *
- * @param innerIO Inner IO.
- * @param leafIO Leaf IO.
- */
- public static void registerTest(BPlusInnerIO<?> innerIO, BPlusLeafIO<?> leafIO) {
- innerTestIO = innerIO;
- leafTestIO = leafIO;
- }
-
- /**
- * @return Type.
- */
- public final int getType() {
- return type;
- }
-
- /**
- * @return Version.
- */
- public final int getVersion() {
- return ver;
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageId Page ID.
- * @param pageSize Page size.
- */
- public void initNewPage(long pageAddr, long pageId, int pageSize) {
- setType(pageAddr, getType());
- setVersion(pageAddr, getVersion());
- setPageId(pageAddr, pageId);
- setCrc(pageAddr, 0);
-
- PageUtils.putLong(pageAddr, RESERVED_1_OFF, 0L);
- PageUtils.putLong(pageAddr, RESERVED_2_OFF, 0L);
- PageUtils.putLong(pageAddr, RESERVED_3_OFF, 0L);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return getClass().getSimpleName() + "[ver=" + getVersion() + "]";
- }
-
- /**
- * @param pageAddr Page address.
- * @return IO.
- * @throws IgniteCheckedException If failed.
- */
- public static <Q extends PageIO> Q getPageIO(long pageAddr) throws IgniteCheckedException {
- int type = getType(pageAddr);
- int ver = getVersion(pageAddr);
-
- return getPageIO(type, ver);
- }
-
- /**
- * @param type IO Type.
- * @param ver IO Version.
- * @return Page IO.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- public static <Q extends PageIO> Q getPageIO(int type, int ver) throws IgniteCheckedException {
- switch (type) {
- case T_DATA:
- return (Q)DataPageIO.VERSIONS.forVersion(ver);
-
- case T_BPLUS_META:
- return (Q)BPlusMetaIO.VERSIONS.forVersion(ver);
-
- case T_PAGE_LIST_NODE:
- return (Q)PagesListNodeIO.VERSIONS.forVersion(ver);
-
- case T_PAGE_LIST_META:
- return (Q)PagesListMetaIO.VERSIONS.forVersion(ver);
-
- case T_META:
- return (Q)PageMetaIO.VERSIONS.forVersion(ver);
-
- case T_PART_META:
- return (Q)PagePartitionMetaIO.VERSIONS.forVersion(ver);
-
- case T_PART_CNTRS:
- return (Q)PagePartitionCountersIO.VERSIONS.forVersion(ver);
-
- case T_PAGE_UPDATE_TRACKING:
- return (Q)TrackingPageIO.VERSIONS.forVersion(ver);
-
- default:
- return (Q)getBPlusIO(type, ver);
- }
- }
-
- /**
- * @param pageAddr Page address.
- * @return IO for either inner or leaf B+Tree page.
- * @throws IgniteCheckedException If failed.
- */
- public static <Q extends BPlusIO<?>> Q getBPlusIO(long pageAddr) throws IgniteCheckedException {
- int type = getType(pageAddr);
- int ver = getVersion(pageAddr);
-
- return getBPlusIO(type, ver);
- }
-
- /**
- * @param type IO Type.
- * @param ver IO Version.
- * @return IO for either inner or leaf B+Tree page.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("unchecked")
- public static <Q extends BPlusIO<?>> Q getBPlusIO(int type, int ver) throws IgniteCheckedException {
-
- if (type >= T_H2_EX_REF_LEAF_START && type <= T_H2_EX_REF_LEAF_END)
- return (Q)h2ExtraLeafIOs.get(type - T_H2_EX_REF_LEAF_START).forVersion(ver);
-
- if (type >= T_H2_EX_REF_INNER_START && type <= T_H2_EX_REF_INNER_END)
- return (Q)h2ExtraInnerIOs.get(type - T_H2_EX_REF_INNER_START).forVersion(ver);
-
- switch (type) {
- case T_H2_REF_INNER:
- if (h2InnerIOs == null)
- break;
-
- return (Q)h2InnerIOs.forVersion(ver);
-
- case T_H2_REF_LEAF:
- if (h2LeafIOs == null)
- break;
-
- return (Q)h2LeafIOs.forVersion(ver);
-
- case T_DATA_REF_INNER:
- return (Q)IgniteCacheOffheapManagerImpl.DataInnerIO.VERSIONS.forVersion(ver);
-
- case T_DATA_REF_LEAF:
- return (Q)IgniteCacheOffheapManagerImpl.DataLeafIO.VERSIONS.forVersion(ver);
-
- case T_CACHE_ID_AWARE_DATA_REF_INNER:
- return (Q)IgniteCacheOffheapManagerImpl.CacheIdAwareDataInnerIO.VERSIONS.forVersion(ver);
-
- case T_CACHE_ID_AWARE_DATA_REF_LEAF:
- return (Q)IgniteCacheOffheapManagerImpl.CacheIdAwareDataLeafIO.VERSIONS.forVersion(ver);
-
- case T_METASTORE_INNER:
- return (Q)MetadataStorage.MetaStoreInnerIO.VERSIONS.forVersion(ver);
-
- case T_METASTORE_LEAF:
- return (Q)MetadataStorage.MetaStoreLeafIO.VERSIONS.forVersion(ver);
-
- case T_PENDING_REF_INNER:
- return (Q) IgniteCacheOffheapManagerImpl.PendingEntryInnerIO.VERSIONS.forVersion(ver);
-
- case T_PENDING_REF_LEAF:
- return (Q)IgniteCacheOffheapManagerImpl.PendingEntryLeafIO.VERSIONS.forVersion(ver);
-
- case T_CACHE_ID_AWARE_PENDING_REF_INNER:
- return (Q) IgniteCacheOffheapManagerImpl.CacheIdAwarePendingEntryInnerIO.VERSIONS.forVersion(ver);
-
- case T_CACHE_ID_AWARE_PENDING_REF_LEAF:
- return (Q)IgniteCacheOffheapManagerImpl.CacheIdAwarePendingEntryLeafIO.VERSIONS.forVersion(ver);
-
- default:
- // For tests.
- if (innerTestIO != null && innerTestIO.getType() == type && innerTestIO.getVersion() == ver)
- return (Q)innerTestIO;
-
- if (leafTestIO != null && leafTestIO.getType() == type && leafTestIO.getVersion() == ver)
- return (Q)leafTestIO;
- }
-
- throw new IgniteCheckedException("Unknown page IO type: " + type);
- }
-
- /**
- * @param type Type to test.
- * @return {@code True} if data page.
- */
- public static boolean isDataPageType(int type) {
- return type == T_DATA;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageMetaIO.java
deleted file mode 100644
index b04baf3..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PageMetaIO.java
+++ /dev/null
@@ -1,222 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.jetbrains.annotations.NotNull;
-
-/**
- *
- */
-public class PageMetaIO extends PageIO {
- /** */
- private static final int TREE_ROOT_OFF = PageIO.COMMON_HEADER_END + 8;
-
- /** */
- private static final int REUSE_LIST_ROOT_OFF = TREE_ROOT_OFF + 8;
-
- /** Last successful snapshot id offset. */
- private static final int LAST_SUCCESSFUL_SNAPSHOT_ID_OFF = REUSE_LIST_ROOT_OFF + 8;
-
- /** Last successful full snapshot id offset. */
- private static final int LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF = LAST_SUCCESSFUL_SNAPSHOT_ID_OFF + 8;
-
- /** Next snapshot id offset. */
- private static final int NEXT_SNAPSHOT_TAG_OFF = LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF + 8;
-
- /** Last successful full snapshot tag offset. */
- private static final int LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF = NEXT_SNAPSHOT_TAG_OFF + 8;
-
- /** Last allocated index offset. */
- private static final int LAST_PAGE_COUNT_OFF = LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF + 8;
-
- /** Candidate allocated index offset. */
- private static final int CANDIDATE_PAGE_COUNT_OFF = LAST_PAGE_COUNT_OFF + 4;
-
- /** End of page meta. */
- static final int END_OF_PAGE_META = CANDIDATE_PAGE_COUNT_OFF + 4;
-
- /** */
- public static final IOVersions<PageMetaIO> VERSIONS = new IOVersions<>(
- new PageMetaIO(1)
- );
-
- /**
- * @param ver Page format version.
- */
- public PageMetaIO(int ver) {
- super(PageIO.T_META, ver);
- }
-
- /**
- * @param type Type.
- * @param ver Version.
- */
- protected PageMetaIO(int type, int ver) {
- super(type, ver);
- }
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setTreeRoot(pageAddr, 0);
- setReuseListRoot(pageAddr, 0);
- setLastSuccessfulFullSnapshotId(pageAddr, 0);
- setLastSuccessfulSnapshotId(pageAddr, 0);
- setNextSnapshotTag(pageAddr, 1);
- setLastSuccessfulSnapshotTag(pageAddr, 0);
- setLastPageCount(pageAddr, 0);
- setCandidatePageCount(pageAddr, 0);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Tree root page.
- */
- public long getTreeRoot(long pageAddr) {
- return PageUtils.getLong(pageAddr, TREE_ROOT_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param treeRoot Tree root
- */
- public void setTreeRoot(long pageAddr, long treeRoot) {
- PageUtils.putLong(pageAddr, TREE_ROOT_OFF, treeRoot);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Reuse list root page.
- */
- public long getReuseListRoot(long pageAddr) {
- return PageUtils.getLong(pageAddr, REUSE_LIST_ROOT_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageId Root page ID.
- */
- public void setReuseListRoot(long pageAddr, long pageId) {
- PageUtils.putLong(pageAddr, REUSE_LIST_ROOT_OFF, pageId);
- }
-
- /**
- * @param pageAddr Page address.
- * @param lastSuccessfulSnapshotId Last successful snapshot id.
- */
- public void setLastSuccessfulSnapshotId(long pageAddr, long lastSuccessfulSnapshotId) {
- PageUtils.putLong(pageAddr, LAST_SUCCESSFUL_SNAPSHOT_ID_OFF, lastSuccessfulSnapshotId);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public long getLastSuccessfulSnapshotId(long pageAddr) {
- return PageUtils.getLong(pageAddr, LAST_SUCCESSFUL_SNAPSHOT_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param lastSuccessfulFullSnapshotId Last successful full snapshot id.
- */
- public void setLastSuccessfulFullSnapshotId(long pageAddr, long lastSuccessfulFullSnapshotId) {
- PageUtils.putLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF, lastSuccessfulFullSnapshotId);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public long getLastSuccessfulFullSnapshotId(long pageAddr) {
- return PageUtils.getLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param nextSnapshotId Next snapshot id.
- */
- public void setNextSnapshotTag(long pageAddr, long nextSnapshotId) {
- PageUtils.putLong(pageAddr, NEXT_SNAPSHOT_TAG_OFF, nextSnapshotId);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public long getLastSuccessfulSnapshotTag(long pageAddr) {
- return PageUtils.getLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param lastSuccessfulSnapshotTag Last successful snapshot tag.
- */
- public void setLastSuccessfulSnapshotTag(long pageAddr, long lastSuccessfulSnapshotTag) {
- PageUtils.putLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF, lastSuccessfulSnapshotTag);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public long getNextSnapshotTag(long pageAddr) {
- return PageUtils.getLong(pageAddr, NEXT_SNAPSHOT_TAG_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageCnt Last allocated index.
- */
- public void setLastPageCount(long pageAddr, int pageCnt) {
- PageUtils.putInt(pageAddr, LAST_PAGE_COUNT_OFF, pageCnt);
- }
-
- /**
- * @param buf Buffer.
- */
- public int getLastPageCount(@NotNull ByteBuffer buf) {
- return buf.getInt(LAST_PAGE_COUNT_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public int getLastPageCount(long pageAddr) {
- return PageUtils.getInt(pageAddr, LAST_PAGE_COUNT_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageCnt Last page count.
- */
- public boolean setCandidatePageCount(long pageAddr, int pageCnt) {
- if (getCandidatePageCount(pageAddr) == pageCnt)
- return false;
-
- PageUtils.putInt(pageAddr, CANDIDATE_PAGE_COUNT_OFF, pageCnt);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- */
- public int getCandidatePageCount(long pageAddr) {
- return PageUtils.getInt(pageAddr, CANDIDATE_PAGE_COUNT_OFF);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionCountersIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionCountersIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionCountersIO.java
deleted file mode 100644
index 015b8ff..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionCountersIO.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.util.Map;
-import org.apache.ignite.internal.pagemem.PageUtils;
-
-/**
- *
- */
-public class PagePartitionCountersIO extends PageIO {
- /** */
- private static final int CNT_OFF = COMMON_HEADER_END;
-
- /** */
- private static final int LAST_FLAG_OFF = CNT_OFF + 2;
-
- /** */
- private static final int NEXT_COUNTERS_PAGE_OFF = LAST_FLAG_OFF + 1;
-
- /** */
- private static final int ITEMS_OFF = NEXT_COUNTERS_PAGE_OFF + 8;
-
- /** */
- private static final int ITEM_SIZE = 12;
-
- /** */
- private static final byte LAST_FLAG = 0b1;
-
- /** */
- public static final IOVersions<PagePartitionCountersIO> VERSIONS = new IOVersions<>(
- new PagePartitionCountersIO(1)
- );
-
- /**
- * @param ver Page format version.
- */
- public PagePartitionCountersIO(int ver) {
- super(T_PART_CNTRS, ver);
- }
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setCount(pageAddr, 0);
- setNextCountersPageId(pageAddr, 0);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Next counters page ID or {@code 0} if it does not exist.
- */
- public long getNextCountersPageId(long pageAddr) {
- return PageUtils.getLong(pageAddr, NEXT_COUNTERS_PAGE_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param partMetaPageId Next counters page ID.
- */
- public void setNextCountersPageId(long pageAddr, long partMetaPageId) {
- PageUtils.putLong(pageAddr, NEXT_COUNTERS_PAGE_OFF, partMetaPageId);
- }
-
- /**
- * @param pageSize Page size.
- * @param pageAddr Page address.
- * @param cacheSizes Serialized cache size items (pairs of cache ID and its size).
- * @return Number of written pairs.
- */
- public int writeCacheSizes(int pageSize, long pageAddr, byte[] cacheSizes, int itemsOff) {
- assert cacheSizes != null;
- assert cacheSizes.length % ITEM_SIZE == 0 : cacheSizes.length;
-
- int cap = getCapacity(pageSize);
-
- int items = (cacheSizes.length / ITEM_SIZE) - itemsOff;
- int write = Math.min(cap, items);
-
- if (write > 0)
- // This can happen in case there are no items in a given partition for all caches in the group.
- PageUtils.putBytes(pageAddr, ITEMS_OFF, cacheSizes, itemsOff * ITEM_SIZE, write * ITEM_SIZE);
-
- setCount(pageAddr, write);
-
- setLastFlag(pageAddr, write == items);
-
- return write;
- }
-
- /**
- * @param pageAddr Page address.
- * @param res Result map of cache sizes.
- * @return {@code True} if the map was fully read.
- */
- public boolean readCacheSizes(long pageAddr, Map<Integer, Long> res) {
- int cnt = getCount(pageAddr);
-
- assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
-
- if (cnt == 0)
- return true;
-
- int off = ITEMS_OFF;
-
- for (int i = 0; i < cnt; i++) {
- int cacheId = PageUtils.getInt(pageAddr, off);
- off += 4;
-
- assert cacheId != 0;
-
- long cacheSize = PageUtils.getLong(pageAddr, off);
- off += 8;
-
- assert cacheSize > 0 : cacheSize;
-
- Long old = res.put(cacheId, cacheSize);
-
- assert old == null;
- }
-
- return getLastFlag(pageAddr);
- }
-
- private boolean getLastFlag(long pageAddr) {
- return PageUtils.getByte(pageAddr, LAST_FLAG_OFF) == LAST_FLAG;
- }
-
- private void setLastFlag(long pageAddr, boolean last) {
- PageUtils.putByte(pageAddr, LAST_FLAG_OFF, last ? LAST_FLAG : ~LAST_FLAG);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Stored items count.
- */
- private int getCount(long pageAddr) {
- return PageUtils.getShort(pageAddr, CNT_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Stored items count.
- */
- private void setCount(long pageAddr, int cnt) {
- assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
-
- PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
- }
-
- /**
- * @param pageSize Page size.
- * @return Maximum number of items which can be stored in buffer.
- */
- private int getCapacity(int pageSize) {
- return (pageSize - ITEMS_OFF) / ITEM_SIZE;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionMetaIO.java
deleted file mode 100644
index ddacf69..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/PagePartitionMetaIO.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import org.apache.ignite.internal.pagemem.PageUtils;
-
-/**
- *
- */
-public class PagePartitionMetaIO extends PageMetaIO {
- /** */
- private static final int SIZE_OFF = PageMetaIO.END_OF_PAGE_META;
-
- /** */
- private static final int UPDATE_CNTR_OFF = SIZE_OFF + 8;
-
- /** */
- private static final int GLOBAL_RMV_ID_OFF = UPDATE_CNTR_OFF + 8;
-
- /** */
- private static final int PARTITION_STATE_OFF = GLOBAL_RMV_ID_OFF + 8;
-
- /** */
- private static final int NEXT_PART_META_PAGE_OFF = PARTITION_STATE_OFF + 1;
-
- /** */
- public static final IOVersions<PagePartitionMetaIO> VERSIONS = new IOVersions<>(
- new PagePartitionMetaIO(1)
- );
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setSize(pageAddr, 0);
- setUpdateCounter(pageAddr, 0);
- setGlobalRemoveId(pageAddr, 0);
- setPartitionState(pageAddr, (byte)-1);
- setCountersPageId(pageAddr, 0);
- }
-
- /**
- * @param ver Version.
- */
- public PagePartitionMetaIO(int ver) {
- super(T_PART_META, ver);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Partition size.
- */
- public long getSize(long pageAddr) {
- return PageUtils.getLong(pageAddr, SIZE_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param size Partition size.
- */
- public boolean setSize(long pageAddr, long size) {
- if (getSize(pageAddr) == size)
- return false;
-
- PageUtils.putLong(pageAddr, SIZE_OFF, size);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Partition update counter.
- */
- public long getUpdateCounter(long pageAddr) {
- return PageUtils.getLong(pageAddr, UPDATE_CNTR_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param cntr Partition update counter.
- */
- public boolean setUpdateCounter(long pageAddr, long cntr) {
- if (getUpdateCounter(pageAddr) == cntr)
- return false;
-
- PageUtils.putLong(pageAddr, UPDATE_CNTR_OFF, cntr);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Global remove ID.
- */
- public long getGlobalRemoveId(long pageAddr) {
- return PageUtils.getLong(pageAddr, GLOBAL_RMV_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param rmvId Global remove ID.
- */
- public boolean setGlobalRemoveId(long pageAddr, long rmvId) {
- if (getGlobalRemoveId(pageAddr) == rmvId)
- return false;
-
- PageUtils.putLong(pageAddr, GLOBAL_RMV_ID_OFF, rmvId);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- */
- public byte getPartitionState(long pageAddr) {
- return PageUtils.getByte(pageAddr, PARTITION_STATE_OFF);
- }
-
- /**
- * @param pageAddr Page address
- * @param state State.
- */
- public boolean setPartitionState(long pageAddr, byte state) {
- if (getPartitionState(pageAddr) == state)
- return false;
-
- PageUtils.putByte(pageAddr, PARTITION_STATE_OFF, state);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Next meta partial page ID or {@code 0} if it does not exist.
- */
- public long getCountersPageId(long pageAddr) {
- return PageUtils.getLong(pageAddr, NEXT_PART_META_PAGE_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param metaPageId Next partial meta page ID.
- */
- public void setCountersPageId(long pageAddr, long metaPageId) {
- PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, metaPageId);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIO.java
deleted file mode 100644
index 136ebff..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIO.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-
-/**
- * We use dedicated page for tracking pages updates.
- * Also we divide such 'tracking' pages on two half, first is used for check that page was changed or not
- * (during incremental snapshot), second - to accumulate changed for next snapshot.
- *
- * You cannot test change for not started snapshot! because it will cause of preparation for snapshot.
- *
- * Implementation. For each page there is own bit in both half. Tracking page is used for tracking N page after it.
- * N depends on page size (how many bytes we can use for tracking).
- *
- *
- * +-----------------------------------------+-----------------------------------------+
- * | left half | right half |
- * +---------+----------+----+------------------------------------+----+------------------------------------+
- * | HEADER | Last |size| |size| |
- * | |SnapshotId|2b. | tracking bits |2b. | tracking bits |
- * +---------+----------+----+------------------------------------+----+------------------------------------+
- *
- */
-public class TrackingPageIO extends PageIO {
- /** */
- public static final IOVersions<TrackingPageIO> VERSIONS = new IOVersions<>(
- new TrackingPageIO(1)
- );
-
- /** Last snapshot offset. */
- public static final int LAST_SNAPSHOT_TAG_OFFSET = COMMON_HEADER_END;
-
- /** Size field offset. */
- public static final int SIZE_FIELD_OFFSET = LAST_SNAPSHOT_TAG_OFFSET + 8;
-
- /** 'Size' field size. */
- public static final int SIZE_FIELD_SIZE = 2;
-
- /** Bitmap offset. */
- public static final int BITMAP_OFFSET = SIZE_FIELD_OFFSET + SIZE_FIELD_SIZE;
-
- /** Count of extra page. */
- public static final int COUNT_OF_EXTRA_PAGE = 1;
-
- /**
- * @param ver Page format version.
- */
- protected TrackingPageIO(int ver) {
- super(PageIO.T_PAGE_UPDATE_TRACKING, ver);
- }
-
- /**
- * Will mark pageId as changed for next (!) snapshotId
- *
- * @param buf Buffer.
- * @param pageId Page id.
- * @param nextSnapshotTag tag of next snapshot.
- * @param pageSize Page size.
- */
- public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
- validateSnapshotId(buf, nextSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
-
- int cntOfPage = countOfPageToTrack(pageSize);
-
- int idxToUpdate = (PageIdUtils.pageIndex(pageId) - COUNT_OF_EXTRA_PAGE) % cntOfPage;
-
- int sizeOff = useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + (cntOfPage >> 3);
-
- int idx = sizeOff + SIZE_FIELD_SIZE + (idxToUpdate >> 3);
-
- byte byteToUpdate = buf.get(idx);
-
- int updateTemplate = 1 << (idxToUpdate & 0b111);
-
- byte newVal = (byte) (byteToUpdate | updateTemplate);
-
- if (byteToUpdate == newVal)
- return false;
-
- buf.put(idx, newVal);
-
- short newSize = (short)(buf.getShort(sizeOff) + 1);
-
- buf.putShort(sizeOff, newSize);
-
- assert newSize == countOfChangedPage(buf, nextSnapshotTag, pageSize);
-
- return true;
- }
-
- /**
- * @param buf Buffer.
- * @param nextSnapshotTag Next snapshot id.
- * @param lastSuccessfulSnapshotId Last successful snapshot id.
- * @param pageSize Page size.
- */
- private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastSuccessfulSnapshotId, int pageSize) {
- assert nextSnapshotTag != lastSuccessfulSnapshotId : "nextSnapshotTag = " + nextSnapshotTag +
- ", lastSuccessfulSnapshotId = " + lastSuccessfulSnapshotId;
-
- long last = getLastSnapshotTag(buf);
-
- assert last <= nextSnapshotTag : "last = " + last + ", nextSnapshotTag = " + nextSnapshotTag;
-
- if (nextSnapshotTag == last) //everything is ok
- return;
-
- int cntOfPage = countOfPageToTrack(pageSize);
-
- if (last <= lastSuccessfulSnapshotId) { //we can drop our data
- buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
-
- PageHandler.zeroMemory(buf, SIZE_FIELD_OFFSET, buf.capacity() - SIZE_FIELD_OFFSET);
- } else { //we can't drop data, it is still necessary for incremental snapshots
- int len = cntOfPage >> 3;
-
- int sizeOff = useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + len;
- int sizeOff2 = !useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + len;
-
- if (last - lastSuccessfulSnapshotId == 1) { //we should keep only data in last half
- //new data will be written in the same half, we should move old data to another half
- if ((nextSnapshotTag - last) % 2 == 0)
- PageHandler.copyMemory(buf, buf, sizeOff, sizeOff2, len + SIZE_FIELD_SIZE);
- } else { //last - lastSuccessfulSnapshotId > 1, e.g. we should merge two half in one
- int newSize = 0;
- int i = 0;
-
- for (; i < len - 8; i += 8) {
- long newVal = buf.getLong(sizeOff + SIZE_FIELD_SIZE + i) | buf.getLong(sizeOff2 + SIZE_FIELD_SIZE + i);
-
- newSize += Long.bitCount(newVal);
-
- buf.putLong(sizeOff2 + SIZE_FIELD_SIZE + i, newVal);
- }
-
- for (; i < len; i ++) {
- byte newVal = (byte) (buf.get(sizeOff + SIZE_FIELD_SIZE + i) | buf.get(sizeOff2 + SIZE_FIELD_SIZE + i));
-
- newSize += Integer.bitCount(newVal & 0xFF);
-
- buf.put(sizeOff2 + SIZE_FIELD_SIZE + i, newVal);
- }
-
- buf.putShort(sizeOff2, (short)newSize);
- }
-
- buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
-
- PageHandler.zeroMemory(buf, sizeOff, len + SIZE_FIELD_SIZE);
- }
- }
-
- /**
- * @param buf Buffer.
- */
- long getLastSnapshotTag(ByteBuffer buf) {
- return buf.getLong(LAST_SNAPSHOT_TAG_OFFSET);
- }
-
- /**
- * Check that pageId was marked as changed between previous snapshot finish and current snapshot start.
- *
- * @param buf Buffer.
- * @param pageId Page id.
- * @param curSnapshotTag Snapshot tag.
- * @param pageSize Page size.
- */
- public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
- validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
-
- if (countOfChangedPage(buf, curSnapshotTag, pageSize) < 1)
- return false;
-
- int cntOfPage = countOfPageToTrack(pageSize);
-
- int idxToTest = (PageIdUtils.pageIndex(pageId) - COUNT_OF_EXTRA_PAGE) % cntOfPage;
-
- byte byteToTest;
-
- if (useLeftHalf(curSnapshotTag))
- byteToTest = buf.get(BITMAP_OFFSET + (idxToTest >> 3));
- else
- byteToTest = buf.get(BITMAP_OFFSET + SIZE_FIELD_SIZE + ((idxToTest + cntOfPage) >> 3));
-
- int testTemplate = 1 << (idxToTest & 0b111);
-
- return ((byteToTest & testTemplate) ^ testTemplate) == 0;
- }
-
- /**
- * @param buf Buffer.
- * @param snapshotTag Snapshot tag.
- * @param pageSize Page size.
- *
- * @return count of pages which were marked as change for given snapshotTag
- */
- public short countOfChangedPage(ByteBuffer buf, long snapshotTag, int pageSize) {
- long dif = getLastSnapshotTag(buf) - snapshotTag;
-
- if (dif != 0 && dif != 1)
- return -1;
-
- if (useLeftHalf(snapshotTag))
- return buf.getShort(SIZE_FIELD_OFFSET);
- else
- return buf.getShort(BITMAP_OFFSET + (countOfPageToTrack(pageSize) >> 3));
- }
-
- /**
- * @param snapshotTag Snapshot id.
- *
- * @return true if snapshotTag is odd, otherwise - false
- */
- boolean useLeftHalf(long snapshotTag) {
- return (snapshotTag & 0b1) == 0;
- }
-
- /**
- * @param pageId Page id.
- * @param pageSize Page size.
- * @return pageId of tracking page which set pageId belongs to
- */
- public long trackingPageFor(long pageId, int pageSize) {
- assert PageIdUtils.pageIndex(pageId) > 0;
-
- int pageIdx = ((PageIdUtils.pageIndex(pageId) - COUNT_OF_EXTRA_PAGE) /
- countOfPageToTrack(pageSize)) * countOfPageToTrack(pageSize) + COUNT_OF_EXTRA_PAGE;
-
- long trackingPageId = PageIdUtils.pageId(PageIdUtils.partId(pageId), PageIdUtils.flag(pageId), pageIdx);
-
- assert PageIdUtils.pageIndex(trackingPageId) <= PageIdUtils.pageIndex(pageId);
-
- return trackingPageId;
- }
-
- /**
- * @param pageSize Page size.
- *
- * @return how many page we can track with 1 page
- */
- public int countOfPageToTrack(int pageSize) {
- return ((pageSize - SIZE_FIELD_OFFSET) / 2 - SIZE_FIELD_SIZE) << 3;
- }
-
- /**
- * @param buf Buffer.
- * @param start Start.
- * @param curSnapshotTag Snapshot id.
- * @param pageSize Page size.
- * @return set pageId if it was changed or next closest one, if there is no changed page null will be returned
- */
- public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
- validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
-
- int cntOfPage = countOfPageToTrack(pageSize);
-
- long trackingPage = trackingPageFor(start, pageSize);
-
- if (start == trackingPage)
- return trackingPage;
-
- if (countOfChangedPage(buf, curSnapshotTag, pageSize) <= 0)
- return null;
-
- int idxToStartTest = (PageIdUtils.pageIndex(start) - COUNT_OF_EXTRA_PAGE) % cntOfPage;
-
- int zeroIdx = useLeftHalf(curSnapshotTag)? BITMAP_OFFSET : BITMAP_OFFSET + SIZE_FIELD_SIZE + (cntOfPage >> 3);
-
- int startIdx = zeroIdx + (idxToStartTest >> 3);
-
- int idx = startIdx;
-
- int stopIdx = zeroIdx + (cntOfPage >> 3);
-
- while (idx < stopIdx) {
- byte byteToTest = buf.get(idx);
-
- if (byteToTest != 0) {
- int foundSetBit;
- if ((foundSetBit = foundSetBit(byteToTest, idx == startIdx ? (idxToStartTest & 0b111) : 0)) != -1) {
- long foundPageId = PageIdUtils.pageId(
- PageIdUtils.partId(start),
- PageIdUtils.flag(start),
- PageIdUtils.pageIndex(trackingPage) + ((idx - zeroIdx) << 3) + foundSetBit);
-
- assert wasChanged(buf, foundPageId, curSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
- assert trackingPageFor(foundPageId, pageSize) == trackingPage;
-
- return foundPageId;
- }
- }
-
- idx++;
- }
-
- return null;
- }
-
- /**
- * @param byteToTest Byte to test.
- * @param firstBitToTest First bit to test.
- */
- private static int foundSetBit(byte byteToTest, int firstBitToTest) {
- assert firstBitToTest < 8;
-
- for (int i = firstBitToTest; i < 8; i++) {
- int testTemplate = 1 << i;
-
- if (((byteToTest & testTemplate) ^ testTemplate) == 0)
- return i;
- }
-
- return -1;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java
deleted file mode 100644
index 903e93f..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseBag.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.reuse;
-
-/**
- * Reuse bag for free index pages.
- */
-public interface ReuseBag {
- /**
- * @param pageId Free page ID for reuse.
- */
- public void addFreePage(long pageId);
-
- /**
- * @return Free page ID for reuse or {@code 0} if empty.
- */
- public long pollFreePage();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
deleted file mode 100644
index 801f89a..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseList.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.reuse;
-
-import org.apache.ignite.IgniteCheckedException;
-
-/**
- * Reuse list.
- */
-public interface ReuseList {
- /**
- * @param bag Reuse bag.
- * @throws IgniteCheckedException If failed.
- */
- public void addForRecycle(ReuseBag bag) throws IgniteCheckedException;
-
- /**
- * @return Page ID or {@code 0} if none available.
- * @throws IgniteCheckedException If failed.
- */
- public long takeRecycledPage() throws IgniteCheckedException;
-
- /**
- * @return Number of recycled pages it contains.
- * @throws IgniteCheckedException If failed.
- */
- public long recycledPagesCount() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseListImpl.java
deleted file mode 100644
index 1c6c8d5..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/reuse/ReuseListImpl.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.reuse;
-
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.database.freelist.PagesList;
-
-/**
- * Reuse list.
- */
-public class ReuseListImpl extends PagesList implements ReuseList {
- /** */
- private static final AtomicReferenceFieldUpdater<ReuseListImpl, Stripe[]> bucketUpdater =
- AtomicReferenceFieldUpdater.newUpdater(ReuseListImpl.class, Stripe[].class, "bucket");
-
- /** */
- private volatile Stripe[] bucket;
-
- /**
- * @param cacheId Cache ID.
- * @param name Name (for debug purpose).
- * @param pageMem Page memory.
- * @param wal Write ahead log manager.
- * @param metaPageId Metadata page ID.
- * @param initNew {@code True} if new metadata should be initialized.
- * @throws IgniteCheckedException If failed.
- */
- public ReuseListImpl(int cacheId,
- String name,
- PageMemory pageMem,
- IgniteWriteAheadLogManager wal,
- long metaPageId,
- boolean initNew) throws IgniteCheckedException {
- super(cacheId, name, pageMem, 1, wal, metaPageId);
-
- reuseList = this;
-
- init(metaPageId, initNew);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean isReuseBucket(int bucket) {
- assert bucket == 0: bucket;
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
- put(bag, 0, 0, 0, 0);
- }
-
- /** {@inheritDoc} */
- @Override public long takeRecycledPage() throws IgniteCheckedException {
- return takeEmptyPage(0, null);
- }
-
- /** {@inheritDoc} */
- @Override public long recycledPagesCount() throws IgniteCheckedException {
- return storedPagesCount(0);
- }
-
- /** {@inheritDoc} */
- @Override protected Stripe[] getBucket(int bucket) {
- return this.bucket;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
- return bucketUpdater.compareAndSet(this, exp, upd);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "ReuseList [name=" + name + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
deleted file mode 100644
index 0d331e7..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageHandler.java
+++ /dev/null
@@ -1,511 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.util;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.util.GridUnsafe;
-
-import static java.lang.Boolean.FALSE;
-import static java.lang.Boolean.TRUE;
-
-/**
- * Page handler.
- */
-public abstract class PageHandler<X, R> {
- /** */
- private static final PageHandler<Void, Boolean> NO_OP = new PageHandler<Void, Boolean>() {
- @Override public Boolean run(int cacheId, long pageId, long page, long pageAddr, PageIO io, Boolean walPlc,
- Void arg,
- int intArg) throws IgniteCheckedException {
- return Boolean.TRUE;
- }
- };
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page absolute pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param walPlc Full page WAL record policy.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @return Result.
- * @throws IgniteCheckedException If failed.
- */
- public abstract R run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO io,
- Boolean walPlc,
- X arg,
- int intArg)
- throws IgniteCheckedException;
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @return {@code true} If release.
- */
- public boolean releaseAfterWrite(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- X arg,
- int intArg) {
- return true;
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param lsnr Lock listener.
- * @param h Handler.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- public static <X, R> R readPage(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- PageLockListener lsnr,
- PageHandler<X, R> h,
- X arg,
- int intArg,
- R lockFailed
- ) throws IgniteCheckedException {
- long page = pageMem.acquirePage(cacheId, pageId);
- try {
- long pageAddr = readLock(pageMem, cacheId, pageId, page, lsnr);
-
- if (pageAddr == 0L)
- return lockFailed;
- try {
- PageIO io = PageIO.getPageIO(pageAddr);
- return h.run(cacheId, pageId, page, pageAddr, io, null, arg, intArg);
- }
- finally {
- readUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr);
- }
- }
- finally {
- pageMem.releasePage(cacheId, pageId, page);
- }
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param lsnr Lock listener.
- * @param h Handler.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- public static <X, R> R readPage(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- PageLockListener lsnr,
- PageHandler<X, R> h,
- X arg,
- int intArg,
- R lockFailed
- ) throws IgniteCheckedException {
- long pageAddr = readLock(pageMem, cacheId, pageId, page, lsnr);
-
- if (pageAddr == 0L)
- return lockFailed;
- try {
- PageIO io = PageIO.getPageIO(pageAddr);
- return h.run(cacheId, pageId, page, pageAddr, io, null, arg, intArg);
- }
- finally {
- readUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr);
- }
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param lsnr Lock listener.
- * @return Page address.
- */
- public static long readLock(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- PageLockListener lsnr) {
- lsnr.onBeforeReadLock(cacheId, pageId, page);
-
- long pageAddr = pageMem.readLock(cacheId, pageId, page);
-
- lsnr.onReadLock(cacheId, pageId, page, pageAddr);
-
- return pageAddr;
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address (for-write pointer)
- * @param lsnr Lock listener.
- */
- public static void readUnlock(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageLockListener lsnr) {
- lsnr.onReadUnlock(cacheId, pageId, page, pageAddr);
-
- pageMem.readUnlock(cacheId, pageId, page);
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param init IO for new page initialization.
- * @param wal Write ahead log.
- * @param lsnr Lock listener.
- * @throws IgniteCheckedException If failed.
- */
- public static void initPage(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- PageIO init,
- IgniteWriteAheadLogManager wal,
- PageLockListener lsnr
- ) throws IgniteCheckedException {
- Boolean res = writePage(pageMem, cacheId, pageId, lsnr, PageHandler.NO_OP, init, wal, null, null, 0, FALSE);
-
- assert res != FALSE;
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param lsnr Lock listener.
- * @param h Handler.
- * @param init IO for new page initialization or {@code null} if it is an existing page.
- * @param wal Write ahead log.
- * @param walPlc Full page WAL record policy.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- public static <X, R> R writePage(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- PageLockListener lsnr,
- PageHandler<X, R> h,
- PageIO init,
- IgniteWriteAheadLogManager wal,
- Boolean walPlc,
- X arg,
- int intArg,
- R lockFailed
- ) throws IgniteCheckedException {
- boolean releaseAfterWrite = true;
- long page = pageMem.acquirePage(cacheId, pageId);
- try {
- long pageAddr = writeLock(pageMem, cacheId, pageId, page, lsnr, false);
-
- if (pageAddr == 0L)
- return lockFailed;
-
- boolean ok = false;
-
- try {
- if (init != null) {
- // It is a new page and we have to initialize it.
- doInitPage(pageMem, cacheId, pageId, page, pageAddr, init, wal);
- walPlc = FALSE;
- }
- else {
- init = PageIO.getPageIO(pageAddr);
- }
-
- R res = h.run(cacheId, pageId, page, pageAddr, init, walPlc, arg, intArg);
-
- ok = true;
-
- return res;
- }
- finally {
- assert PageIO.getCrc(pageAddr) == 0; //TODO GG-11480
-
- if (releaseAfterWrite = h.releaseAfterWrite(cacheId, pageId, page, pageAddr, arg, intArg))
- writeUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr, walPlc, ok);
- }
- }
- finally {
- if (releaseAfterWrite)
- pageMem.releasePage(cacheId, pageId, page);
- }
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param lsnr Lock listener.
- * @param h Handler.
- * @param init IO for new page initialization or {@code null} if it is an existing page.
- * @param wal Write ahead log.
- * @param walPlc Full page WAL record policy.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- public static <X, R> R writePage(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- PageLockListener lsnr,
- PageHandler<X, R> h,
- PageIO init,
- IgniteWriteAheadLogManager wal,
- Boolean walPlc,
- X arg,
- int intArg,
- R lockFailed
- ) throws IgniteCheckedException {
- long pageAddr = writeLock(pageMem, cacheId, pageId, page, lsnr, false);
-
- if (pageAddr == 0L)
- return lockFailed;
-
- boolean ok = false;
-
- try {
- if (init != null) {
- // It is a new page and we have to initialize it.
- doInitPage(pageMem, cacheId, pageId, page, pageAddr, init, wal);
- walPlc = FALSE;
- }
- else {
- init = PageIO.getPageIO(pageAddr);
- }
-
- R res = h.run(cacheId, pageId, page, pageAddr, init, walPlc, arg, intArg);
-
- ok = true;
-
- return res;
- }
- finally {
- assert PageIO.getCrc(pageAddr) == 0; //TODO GG-11480
-
- if (h.releaseAfterWrite(cacheId, pageId, page, pageAddr, arg, intArg))
- writeUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr, walPlc, ok);
- }
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param lsnr Lock listener.
- * @param walPlc Full page WAL record policy.
- * @param dirty Page is dirty.
- */
- public static void writeUnlock(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageLockListener lsnr,
- Boolean walPlc,
- boolean dirty) {
- lsnr.onWriteUnlock(cacheId, pageId, page, pageAddr);
-
- pageMem.writeUnlock(cacheId, pageId, page, walPlc, dirty);
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param lsnr Lock listener.
- * @param tryLock Only try to lock without waiting.
- * @return Page address or {@code 0} if failed to lock due to recycling.
- */
- public static long writeLock(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- PageLockListener lsnr,
- boolean tryLock) {
- lsnr.onBeforeWriteLock(cacheId, pageId, page);
-
- long pageAddr = tryLock ? pageMem.tryWriteLock(cacheId, pageId, page) : pageMem.writeLock(cacheId, pageId, page);
-
- lsnr.onWriteLock(cacheId, pageId, page, pageAddr);
-
- return pageAddr;
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param init Initial IO.
- * @param wal Write ahead log.
- * @throws IgniteCheckedException If failed.
- */
- private static void doInitPage(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO init,
- IgniteWriteAheadLogManager wal) throws IgniteCheckedException {
-
- assert PageIO.getCrc(pageAddr) == 0; //TODO GG-11480
-
- init.initNewPage(pageAddr, pageId, pageMem.pageSize());
-
- // Here we should never write full page, because it is known to be new.
- if (isWalDeltaRecordNeeded(pageMem, cacheId, pageId, page, wal, FALSE))
- wal.log(new InitNewPageRecord(cacheId, pageId,
- init.getType(), init.getVersion(), pageId));
- }
-
- /**
- * @param pageMem Page memory.
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param wal Write ahead log.
- * @param walPlc Full page WAL record policy.
- * @return {@code true} If we need to make a delta WAL record for the change in this page.
- */
- public static boolean isWalDeltaRecordNeeded(
- PageMemory pageMem,
- int cacheId,
- long pageId,
- long page,
- IgniteWriteAheadLogManager wal,
- Boolean walPlc) {
- // If the page is clean, then it is either newly allocated or just after checkpoint.
- // In both cases we have to write full page contents to WAL.
- return wal != null && !wal.isAlwaysWriteFullPages() && walPlc != TRUE &&
- (walPlc == FALSE || pageMem.isDirty(cacheId, pageId, page));
- }
-
- /**
- * @param src Source.
- * @param dst Destination.
- * @param srcOff Source offset in bytes.
- * @param dstOff Destination offset in bytes.
- * @param cnt Bytes count to copy.
- */
- public static void copyMemory(ByteBuffer src, ByteBuffer dst, long srcOff, long dstOff, long cnt) {
- byte[] srcArr = src.hasArray() ? src.array() : null;
- byte[] dstArr = dst.hasArray() ? dst.array() : null;
- long srcArrOff = src.hasArray() ? src.arrayOffset() + GridUnsafe.BYTE_ARR_OFF : 0;
- long dstArrOff = dst.hasArray() ? dst.arrayOffset() + GridUnsafe.BYTE_ARR_OFF : 0;
-
- long srcPtr = src.isDirect() ? GridUnsafe.bufferAddress(src) : 0;
- long dstPtr = dst.isDirect() ? GridUnsafe.bufferAddress(dst) : 0;
-
- GridUnsafe.copyMemory(srcArr, srcPtr + srcArrOff + srcOff, dstArr, dstPtr + dstArrOff + dstOff, cnt);
- }
-
- /**
- * Will zero memory in buf
- * @param buf Buffer.
- * @param off Offset.
- * @param len Length.
- */
- public static void zeroMemory(ByteBuffer buf, int off, int len) {
- if (buf.isDirect())
- GridUnsafe.setMemory(GridUnsafe.bufferAddress(buf) + off, len, (byte)0);
-
- else {
- for (int i = off; i < off + len; i++)
- buf.put(i, (byte)0); //TODO Optimize!
- }
- }
-
- /**
- * @param srcAddr Source.
- * @param dstAddr Destination.
- * @param srcOff Source offset in bytes.
- * @param dstOff Destination offset in bytes.
- * @param cnt Bytes count to copy.
- */
- public static void copyMemory(long srcAddr, long dstAddr, long srcOff, long dstOff, long cnt) {
- GridUnsafe.copyMemory(null, srcAddr + srcOff, null, dstAddr + dstOff, cnt);
- }
-
- /**
- * @param addr Address.
- * @param off Offset.
- * @param len Length.
- */
- public static void zeroMemory(long addr, int off, int len) {
- GridUnsafe.setMemory(addr + off, len, (byte)0);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageLockListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageLockListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageLockListener.java
deleted file mode 100644
index 904d217..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/util/PageLockListener.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.util;
-
-/**
- * Page lock listener.
- */
-public interface PageLockListener {
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- */
- public void onBeforeWriteLock(int cacheId, long pageId, long page);
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- */
- public void onWriteLock(int cacheId, long pageId, long page, long pageAddr);
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- */
- public void onWriteUnlock(int cacheId, long pageId, long page, long pageAddr);
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- */
- public void onBeforeReadLock(int cacheId, long pageId, long page);
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- */
- public void onReadLock(int cacheId, long pageId, long page, long pageAddr);
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- */
- public void onReadUnlock(int cacheId, long pageId, long page, long pageAddr);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
index 4480efe..f0d5bf1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtLocalPartition.java
@@ -47,7 +47,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPreloader;
import org.apache.ignite.internal.processors.cache.extras.GridCacheObsoleteEntryExtras;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index f29c507..78ff595 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -41,7 +41,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedExceptio
import org.apache.ignite.internal.processors.cache.GridCacheMessage;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index aabe025..cab90ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -42,7 +42,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheFutureAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheMessage;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.near.CacheVersionedValue;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
index 2f99033..432e3f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java
@@ -56,13 +56,12 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
-import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtFuture;
[37/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java
new file mode 100644
index 0000000..e490e35
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheSnapshotManager.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.nio.ByteBuffer;
+import java.util.NavigableMap;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ *
+ */
+public class IgniteCacheSnapshotManager extends GridCacheSharedManagerAdapter {
+ /** Snapshot started lock filename. */
+ public static final String SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME = "snapshot-started.loc";
+
+ /**
+ * @param initiatorNodeId Initiator node id.
+ * @param snapshotOperation Snapshot operation.
+ */
+ @Nullable public IgniteInternalFuture startLocalSnapshotOperation(
+ UUID initiatorNodeId,
+ SnapshotOperation snapshotOperation
+ ) throws IgniteCheckedException {
+ return null;
+ }
+
+ /**
+ * @param snapOp current snapshot operation.
+ *
+ * @return {@code true} if next operation must be snapshot, {@code false} if checkpoint must be executed.
+ */
+ public boolean onMarkCheckPointBegin(
+ SnapshotOperation snapOp,
+ NavigableMap<T2<Integer, Integer>, T2<Integer, Integer>> map
+ ) throws IgniteCheckedException {
+ return false;
+ }
+
+ /**
+ *
+ */
+ public void restoreState() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ public void onCheckPointBegin() {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ public void beforeCheckpointPageWritten() {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ public void afterCheckpointPageWritten() {
+ // No-op.
+ }
+
+ /**
+ * @param fullId Full id.
+ */
+ public void beforePageWrite(FullPageId fullId) {
+ // No-op.
+ }
+
+ /**
+ * @param fullId Full id.
+ */
+ public void onPageWrite(FullPageId fullId, ByteBuffer tmpWriteBuf) {
+ // No-op.
+ }
+
+ /**
+ * @param cctx Cctx.
+ */
+ public void onCacheStop(GridCacheContext cctx) {
+ // No-op.
+ }
+
+ /**
+ *
+ */
+ public void onChangeTrackerPage(
+ Long page,
+ FullPageId fullId,
+ PageMemory pageMem
+ ) throws IgniteCheckedException {
+
+ }
+
+ /**
+ *
+ */
+ public void flushDirtyPageHandler(
+ FullPageId fullId,
+ ByteBuffer pageBuf,
+ Integer tag
+ ) throws IgniteCheckedException {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsImpl.java
new file mode 100644
index 0000000..271767c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsImpl.java
@@ -0,0 +1,284 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.MemoryMetrics;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jsr166.LongAdder8;
+
+/**
+ *
+ */
+public class MemoryMetricsImpl implements MemoryMetrics {
+ /** */
+ private FreeListImpl freeList;
+
+ /** */
+ private final LongAdder8 totalAllocatedPages = new LongAdder8();
+
+ /**
+ * Counter for number of pages occupied by large entries (one entry is larger than one page).
+ */
+ private final LongAdder8 largeEntriesPages = new LongAdder8();
+
+ /** Counter for number of dirty pages. */
+ private LongAdder8 dirtyPages = new LongAdder8();
+
+ /** */
+ private volatile boolean metricsEnabled;
+
+ /** */
+ private boolean persistenceEnabled;
+
+ /** */
+ private volatile int subInts;
+
+ /** Allocation rate calculator. */
+ private volatile HitRateMetrics allocRate = new HitRateMetrics(60_000, 5);
+
+ /** */
+ private volatile HitRateMetrics pageReplaceRate = new HitRateMetrics(60_000, 5);
+
+ /** */
+ private final MemoryPolicyConfiguration memPlcCfg;
+
+ /** */
+ private PageMemory pageMem;
+
+ /** Time interval (in milliseconds) when allocations/evictions are counted to calculate rate. */
+ private volatile long rateTimeInterval;
+
+ /**
+ * @param memPlcCfg MemoryPolicyConfiguration.
+ */
+ public MemoryMetricsImpl(MemoryPolicyConfiguration memPlcCfg) {
+ this.memPlcCfg = memPlcCfg;
+
+ metricsEnabled = memPlcCfg.isMetricsEnabled();
+
+ rateTimeInterval = memPlcCfg.getRateTimeInterval();
+
+ subInts = memPlcCfg.getSubIntervals();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getName() {
+ return U.maskName(memPlcCfg.getName());
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedPages() {
+ return metricsEnabled ? totalAllocatedPages.longValue() : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getAllocationRate() {
+ if (!metricsEnabled)
+ return 0;
+
+ return ((float) allocRate.getRate()) / rateTimeInterval;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEvictionRate() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getLargeEntriesPagesPercentage() {
+ if (!metricsEnabled)
+ return 0;
+
+ return totalAllocatedPages.longValue() != 0 ?
+ (float) largeEntriesPages.doubleValue() / totalAllocatedPages.longValue()
+ : 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getPagesFillFactor() {
+ if (!metricsEnabled || freeList == null)
+ return 0;
+
+ return freeList.fillFactor();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getDirtyPages() {
+ if (!metricsEnabled || !persistenceEnabled)
+ return 0;
+
+ return dirtyPages.longValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getPagesReplaceRate() {
+ if (!metricsEnabled || !persistenceEnabled)
+ return 0;
+
+ return ((float) pageReplaceRate.getRate()) / rateTimeInterval;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPhysicalMemoryPages() {
+ if (!metricsEnabled || !persistenceEnabled)
+ return 0;
+
+ assert pageMem != null;
+
+ return pageMem.loadedPages();
+ }
+
+ /**
+ * Updates pageReplaceRate metric.
+ */
+ public void updatePageReplaceRate() {
+ if (metricsEnabled)
+ pageReplaceRate.onHit();
+ }
+
+ /**
+ * Increments dirtyPages counter.
+ */
+ public void incrementDirtyPages() {
+ if (metricsEnabled)
+ dirtyPages.increment();
+ }
+
+ /**
+ * Decrements dirtyPages counter.
+ */
+ public void decrementDirtyPages() {
+ if (metricsEnabled)
+ dirtyPages.decrement();
+ }
+
+ /**
+ * Resets dirtyPages counter to zero.
+ */
+ public void resetDirtyPages() {
+ if (metricsEnabled)
+ dirtyPages.reset();
+ }
+
+ /**
+ * Increments totalAllocatedPages counter.
+ */
+ public void incrementTotalAllocatedPages() {
+ if (metricsEnabled) {
+ totalAllocatedPages.increment();
+
+ updateAllocationRateMetrics();
+ }
+ }
+
+ /**
+ *
+ */
+ private void updateAllocationRateMetrics() {
+ allocRate.onHit();
+ }
+
+ /**
+ * @param intervalNum Interval number.
+ */
+ private long subInt(int intervalNum) {
+ return (rateTimeInterval * intervalNum) / subInts;
+ }
+
+ /**
+ *
+ */
+ public void incrementLargeEntriesPages() {
+ if (metricsEnabled)
+ largeEntriesPages.increment();
+ }
+
+ /**
+ *
+ */
+ public void decrementLargeEntriesPages() {
+ if (metricsEnabled)
+ largeEntriesPages.decrement();
+ }
+
+ /**
+ * Enable metrics.
+ */
+ public void enableMetrics() {
+ metricsEnabled = true;
+ }
+
+ /**
+ * Disable metrics.
+ */
+ public void disableMetrics() {
+ metricsEnabled = false;
+ }
+
+ /**
+ * @param persistenceEnabled Persistence enabled.
+ */
+ public void persistenceEnabled(boolean persistenceEnabled) {
+ this.persistenceEnabled = persistenceEnabled;
+ }
+
+ /**
+ * @param pageMem Page mem.
+ */
+ public void pageMemory(PageMemory pageMem) {
+ this.pageMem = pageMem;
+ }
+
+ /**
+ * @param rateTimeInterval Time interval (in milliseconds) used to calculate allocation/eviction rate.
+ */
+ public void rateTimeInterval(long rateTimeInterval) {
+ this.rateTimeInterval = rateTimeInterval;
+
+ allocRate = new HitRateMetrics((int) rateTimeInterval, subInts);
+ pageReplaceRate = new HitRateMetrics((int) rateTimeInterval, subInts);
+ }
+
+ /**
+ * Sets number of subintervals the whole rateTimeInterval will be split into to calculate allocation rate.
+ *
+ * @param subInts Number of subintervals.
+ */
+ public void subIntervals(int subInts) {
+ assert subInts > 0;
+
+ if (this.subInts == subInts)
+ return;
+
+ if (rateTimeInterval / subInts < 10)
+ subInts = (int) rateTimeInterval / 10;
+
+ allocRate = new HitRateMetrics((int) rateTimeInterval, subInts);
+ pageReplaceRate = new HitRateMetrics((int) rateTimeInterval, subInts);
+ }
+
+ /**
+ * @param freeList Free list.
+ */
+ void freeList(FreeListImpl freeList) {
+ this.freeList = freeList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsMXBeanImpl.java
new file mode 100644
index 0000000..392f83f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsMXBeanImpl.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.MemoryMetrics;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.mxbean.MemoryMetricsMXBean;
+
+/**
+ * MBean to expose {@link MemoryMetrics} through JMX interface.
+ */
+class MemoryMetricsMXBeanImpl implements MemoryMetricsMXBean {
+ /** */
+ private final MemoryMetricsImpl memMetrics;
+
+ /** */
+ private final MemoryPolicyConfiguration memPlcCfg;
+
+ /**
+ * @param memMetrics MemoryMetrics instance to expose through JMX interface.
+ * @param memPlcCfg configuration of memory policy this MX Bean is created for.
+ */
+ MemoryMetricsMXBeanImpl(MemoryMetricsImpl memMetrics,
+ MemoryPolicyConfiguration memPlcCfg
+ ) {
+ this.memMetrics = memMetrics;
+ this.memPlcCfg = memPlcCfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getAllocationRate() {
+ return memMetrics.getAllocationRate();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEvictionRate() {
+ return memMetrics.getEvictionRate();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getLargeEntriesPagesPercentage() {
+ return memMetrics.getLargeEntriesPagesPercentage();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getPagesFillFactor() {
+ return memMetrics.getPagesFillFactor();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedPages() {
+ return memMetrics.getTotalAllocatedPages();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getDirtyPages() {
+ return memMetrics.getDirtyPages();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getPagesReplaceRate() {
+ return memMetrics.getPagesReplaceRate();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPhysicalMemoryPages() {
+ return memMetrics.getPhysicalMemoryPages();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rateTimeInterval(long rateTimeInterval) {
+ if (rateTimeInterval < 1000)
+ throw new IllegalArgumentException("rateTimeInterval property must be positive " +
+ "and greater than 1_000 milliseconds (one second)");
+
+ memMetrics.rateTimeInterval(rateTimeInterval);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void subIntervals(int subInts) {
+ if (subInts <= 1)
+ throw new IllegalArgumentException("subIntervals property must be positive " +
+ "and greater than one");
+
+ memMetrics.subIntervals(subInts);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void enableMetrics() {
+ memMetrics.enableMetrics();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void disableMetrics() {
+ memMetrics.disableMetrics();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getName() {
+ return memMetrics.getName();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getInitialSize() {
+ return (int) (memPlcCfg.getInitialSize() / (1024 * 1024));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getMaxSize() {
+ return (int) (memPlcCfg.getMaxSize() / (1024 * 1024));
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getSwapFilePath() {
+ return memPlcCfg.getSwapFilePath();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsSnapshot.java
new file mode 100644
index 0000000..4e7f90a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryMetricsSnapshot.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.MemoryMetrics;
+
+/**
+ *
+ */
+public class MemoryMetricsSnapshot implements MemoryMetrics {
+ /** */
+ private String name;
+
+ /** */
+ private long totalAllocatedPages;
+
+ /** */
+ private float allocationRate;
+
+ /** */
+ private float evictionRate;
+
+ /** */
+ private float largeEntriesPagesPercentage;
+
+ /** */
+ private float pagesFillFactor;
+
+ /** */
+ private long dirtyPages;
+
+ /** */
+ private float pageReplaceRate;
+
+ /** */
+ private long physicalMemoryPages;
+
+ /**
+ * @param metrics Metrics instance to take a copy.
+ */
+ public MemoryMetricsSnapshot(MemoryMetrics metrics) {
+ name = metrics.getName();
+ totalAllocatedPages = metrics.getTotalAllocatedPages();
+ allocationRate = metrics.getAllocationRate();
+ evictionRate = metrics.getEvictionRate();
+ largeEntriesPagesPercentage = metrics.getLargeEntriesPagesPercentage();
+ pagesFillFactor = metrics.getPagesFillFactor();
+ dirtyPages = metrics.getDirtyPages();
+ pageReplaceRate = metrics.getPagesReplaceRate();
+ physicalMemoryPages = metrics.getPhysicalMemoryPages();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String getName() {
+ return name;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getTotalAllocatedPages() {
+ return totalAllocatedPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getAllocationRate() {
+ return allocationRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getEvictionRate() {
+ return evictionRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getLargeEntriesPagesPercentage() {
+ return largeEntriesPagesPercentage;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getPagesFillFactor() {
+ return pagesFillFactor;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getDirtyPages() {
+ return dirtyPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getPagesReplaceRate() {
+ return pageReplaceRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getPhysicalMemoryPages() {
+ return physicalMemoryPages;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicy.java
new file mode 100644
index 0000000..7c7d7bc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MemoryPolicy.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
+
+/**
+ * Memory policy provides access to objects configured with {@link MemoryPolicyConfiguration} configuration.
+ */
+public class MemoryPolicy {
+ /** */
+ private final PageMemory pageMem;
+
+ /** */
+ private final MemoryMetricsImpl memMetrics;
+
+ /** */
+ private final MemoryPolicyConfiguration cfg;
+
+ /** */
+ private final PageEvictionTracker evictionTracker;
+
+ /**
+ * @param pageMem PageMemory instance.
+ * @param memMetrics MemoryMetrics instance.
+ * @param cfg Configuration of given MemoryPolicy.
+ * @param evictionTracker Eviction tracker.
+ */
+ public MemoryPolicy(
+ PageMemory pageMem,
+ MemoryPolicyConfiguration cfg,
+ MemoryMetricsImpl memMetrics,
+ PageEvictionTracker evictionTracker) {
+ this.pageMem = pageMem;
+ this.memMetrics = memMetrics;
+ this.cfg = cfg;
+ this.evictionTracker = evictionTracker;
+ }
+
+ /**
+ *
+ */
+ public PageMemory pageMemory() {
+ return pageMem;
+ }
+
+ /**
+ * @return Config.
+ */
+ public MemoryPolicyConfiguration config() {
+ return cfg;
+ }
+
+ /**
+ * @return Memory Metrics.
+ */
+ public MemoryMetricsImpl memoryMetrics() {
+ return memMetrics;
+ }
+
+ /**
+ *
+ */
+ public PageEvictionTracker evictionTracker() {
+ return evictionTracker;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
new file mode 100644
index 0000000..c09ce4e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetaStore.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Meta store.
+ */
+public interface MetaStore {
+ /**
+ * Get or allocate initial page for an index.
+ *
+ * @param idxName Index name.
+ * @return {@link RootPage} that keeps pageId, allocated flag that shows whether the page
+ * was newly allocated, and rootId that is counter which increments each time new page allocated.
+ * @throws IgniteCheckedException If failed.
+ */
+ public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException;
+
+ /**
+ * Deallocate index page and remove from tree.
+ *
+ * @param idxName Index name.
+ * @return Root ID or -1 if no page was removed.
+ * @throws IgniteCheckedException If failed.
+ */
+ public RootPage dropRootPage(String idxName) throws IgniteCheckedException;
+
+ /**
+ * Destroy this meta store.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void destroy() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
new file mode 100644
index 0000000..806afb8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/MetadataStorage.java
@@ -0,0 +1,417 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ * Metadata storage.
+ */
+public class MetadataStorage implements MetaStore {
+ /** Max index name length (bytes num) */
+ public static final int MAX_IDX_NAME_LEN = 768;
+
+ /** Bytes in byte. */
+ private static final int BYTE_LEN = 1;
+
+ /** Page memory. */
+ private final PageMemory pageMem;
+
+ /** Index tree. */
+ private final MetaTree metaTree;
+
+ /** Meta page reuse tree. */
+ private final ReuseList reuseList;
+
+ /** Cache group ID. */
+ private final int grpId;
+
+ /** */
+ private final int allocPartId;
+
+ /** */
+ private final byte allocSpace;
+
+ /**
+ * @param pageMem Page memory.
+ * @param wal Write ahead log manager.
+ */
+ public MetadataStorage(
+ final PageMemory pageMem,
+ final IgniteWriteAheadLogManager wal,
+ final AtomicLong globalRmvId,
+ final int grpId,
+ final int allocPartId,
+ final byte allocSpace,
+ final ReuseList reuseList,
+ final long rootPageId,
+ final boolean initNew
+ ) {
+ try {
+ this.pageMem = pageMem;
+ this.grpId = grpId;
+ this.allocPartId = allocPartId;
+ this.allocSpace = allocSpace;
+ this.reuseList = reuseList;
+
+ metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
+ reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException {
+ final MetaTree tree = metaTree;
+
+ synchronized (this) {
+ byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+ if (idxNameBytes.length > MAX_IDX_NAME_LEN)
+ throw new IllegalArgumentException("Too long encoded indexName [maxAllowed=" + MAX_IDX_NAME_LEN +
+ ", currentLength=" + idxNameBytes.length + ", name=" + idxName + "]");
+
+ final IndexItem row = tree.findOne(new IndexItem(idxNameBytes, 0));
+
+ if (row == null) {
+ long pageId = 0;
+
+ if (reuseList != null)
+ pageId = reuseList.takeRecycledPage();
+
+ pageId = pageId == 0 ? pageMem.allocatePage(grpId, allocPartId, allocSpace) : pageId;
+
+ tree.put(new IndexItem(idxNameBytes, pageId));
+
+ return new RootPage(new FullPageId(pageId, grpId), true);
+ }
+ else {
+ final FullPageId pageId = new FullPageId(row.pageId, grpId);
+
+ return new RootPage(pageId, false);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public RootPage dropRootPage(final String idxName)
+ throws IgniteCheckedException {
+ byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
+
+ final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0));
+
+ if (row != null) {
+ if (reuseList == null)
+ pageMem.freePage(grpId, row.pageId);
+ }
+
+ return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void destroy() throws IgniteCheckedException {
+ metaTree.destroy();
+ }
+
+ /**
+ *
+ */
+ private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
+ /** */
+ private final int allocPartId;
+
+ /** */
+ private final byte allocSpace;
+
+ /**
+ * @param pageMem Page memory.
+ * @param metaPageId Meta page ID.
+ * @param reuseList Reuse list.
+ * @param innerIos Inner IOs.
+ * @param leafIos Leaf IOs.
+ * @throws IgniteCheckedException If failed.
+ */
+ private MetaTree(
+ final int cacheId,
+ final int allocPartId,
+ final byte allocSpace,
+ final PageMemory pageMem,
+ final IgniteWriteAheadLogManager wal,
+ final AtomicLong globalRmvId,
+ final long metaPageId,
+ final ReuseList reuseList,
+ final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
+ final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
+ final boolean initNew
+ ) throws IgniteCheckedException {
+ super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
+
+ this.allocPartId = allocPartId;
+ this.allocSpace = allocSpace;
+
+ initTree(initNew);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
+ return pageMem.allocatePage(getCacheId(), allocPartId, allocSpace);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int compare(final BPlusIO<IndexItem> io, final long pageAddr, final int idx,
+ final IndexItem row) throws IgniteCheckedException {
+ final int off = ((IndexIO)io).getOffset(pageAddr, idx);
+
+ int shift = 0;
+
+ // Compare index names.
+ final byte len = PageUtils.getByte(pageAddr, off + shift);
+
+ shift += BYTE_LEN;
+
+ for (int i = 0; i < len && i < row.idxName.length; i++) {
+ final int cmp = Byte.compare(PageUtils.getByte(pageAddr, off + i + shift), row.idxName[i]);
+
+ if (cmp != 0)
+ return cmp;
+ }
+
+ return Integer.compare(len, row.idxName.length);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IndexItem getRow(final BPlusIO<IndexItem> io, final long pageAddr,
+ final int idx, Object ignore) throws IgniteCheckedException {
+ return readRow(pageAddr, ((IndexIO)io).getOffset(pageAddr, idx));
+ }
+ }
+
+ /**
+ *
+ */
+ private static class IndexItem {
+ /** */
+ private byte[] idxName;
+
+ /** */
+ private long pageId;
+
+ /**
+ * @param idxName Index name.
+ * @param pageId Page ID.
+ */
+ private IndexItem(final byte[] idxName, final long pageId) {
+ this.idxName = idxName;
+ this.pageId = pageId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "I [idxName=" + new String(idxName) + ", pageId=" + U.hexLong(pageId) + ']';
+ }
+ }
+
+ /**
+ * Store row to buffer.
+ *
+ * @param pageAddr Page address.
+ * @param off Offset in buf.
+ * @param row Row to store.
+ */
+ private static void storeRow(
+ final long pageAddr,
+ int off,
+ final IndexItem row
+ ) {
+ // Index name length.
+ PageUtils.putByte(pageAddr, off, (byte)row.idxName.length);
+ off++;
+
+ // Index name.
+ PageUtils.putBytes(pageAddr, off, row.idxName);
+ off += row.idxName.length;
+
+ // Page ID.
+ PageUtils.putLong(pageAddr, off, row.pageId);
+ }
+
+ /**
+ * Copy row data.
+ *
+ * @param dstPageAddr Destination page address.
+ * @param dstOff Destination buf offset.
+ * @param srcPageAddr Source page address.
+ * @param srcOff Src buf offset.
+ */
+ private static void storeRow(
+ final long dstPageAddr,
+ int dstOff,
+ final long srcPageAddr,
+ int srcOff
+ ) {
+ // Index name length.
+ final byte len = PageUtils.getByte(srcPageAddr, srcOff);
+ srcOff++;
+
+ PageUtils.putByte(dstPageAddr, dstOff, len);
+ dstOff++;
+
+ PageHandler.copyMemory(srcPageAddr, dstPageAddr, srcOff, dstOff, len);
+ srcOff += len;
+ dstOff += len;
+
+ // Page ID.
+ PageUtils.putLong(dstPageAddr, dstOff, PageUtils.getLong(srcPageAddr, srcOff));
+ }
+
+ /**
+ * Read row from buffer.
+ *
+ * @param pageAddr Page address.
+ * @param off Offset.
+ * @return Read row.
+ */
+ private static IndexItem readRow(final long pageAddr, int off) {
+ // Index name length.
+ final int len = PageUtils.getByte(pageAddr, off) & 0xFF;
+ off++;
+
+ // Index name.
+ final byte[] idxName = PageUtils.getBytes(pageAddr, off, len);
+ off += len;
+
+ // Page ID.
+ final long pageId = PageUtils.getLong(pageAddr, off);
+
+ return new IndexItem(idxName, pageId);
+ }
+
+ /**
+ *
+ */
+ private interface IndexIO {
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Offset in buffer according to {@code idx}.
+ */
+ int getOffset(long pageAddr, int idx);
+ }
+
+ /**
+ *
+ */
+ public static final class MetaStoreInnerIO extends BPlusInnerIO<IndexItem> implements IndexIO {
+ /** */
+ public static final IOVersions<MetaStoreInnerIO> VERSIONS = new IOVersions<>(
+ new MetaStoreInnerIO(1)
+ );
+
+ /**
+ * @param ver Version.
+ */
+ private MetaStoreInnerIO(final int ver) {
+ // name bytes and 1 byte for length, 8 bytes pageId
+ super(T_METASTORE_INNER, ver, false, MAX_IDX_NAME_LEN + 1 + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeByOffset(long pageAddr, int off, IndexItem row) throws IgniteCheckedException {
+ storeRow(pageAddr, off, row);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(final long dstPageAddr, final int dstIdx, final BPlusIO<IndexItem> srcIo,
+ final long srcPageAddr,
+ final int srcIdx) throws IgniteCheckedException {
+ storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree, final long pageAddr,
+ final int idx) throws IgniteCheckedException {
+ return readRow(pageAddr, offset(idx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getOffset(long pageAddr, final int idx) {
+ return offset(idx);
+ }
+ }
+
+ /**
+ *
+ */
+ public static final class MetaStoreLeafIO extends BPlusLeafIO<IndexItem> implements IndexIO {
+ /** */
+ public static final IOVersions<MetaStoreLeafIO> VERSIONS = new IOVersions<>(
+ new MetaStoreLeafIO(1)
+ );
+
+ /**
+ * @param ver Version.
+ */
+ private MetaStoreLeafIO(final int ver) {
+ // 4 byte cache ID, UTF-16 symbols and 1 byte for length, 8 bytes pageId
+ super(T_METASTORE_LEAF, ver, MAX_IDX_NAME_LEN + 1 + 8);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeByOffset(long buf, int off, IndexItem row) throws IgniteCheckedException {
+ storeRow(buf, off, row);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void store(final long dstPageAddr,
+ final int dstIdx,
+ final BPlusIO<IndexItem> srcIo,
+ final long srcPageAddr,
+ final int srcIdx) throws IgniteCheckedException {
+ storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree,
+ final long pageAddr,
+ final int idx) throws IgniteCheckedException {
+ return readRow(pageAddr, offset(idx));
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getOffset(long pageAddr, final int idx) {
+ return offset(idx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsImpl.java
new file mode 100644
index 0000000..7952937
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsImpl.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
+import org.apache.ignite.mxbean.PersistenceMetricsMXBean;
+
+/**
+ *
+ */
+public class PersistenceMetricsImpl implements PersistenceMetricsMXBean {
+ /** */
+ private volatile HitRateMetrics walLoggingRate;
+
+ /** */
+ private volatile HitRateMetrics walWritingRate;
+
+ /** */
+ private volatile HitRateMetrics walFsyncTimeDuration;
+
+ /** */
+ private volatile HitRateMetrics walFsyncTimeNumber;
+
+ /** */
+ private volatile long lastCpLockWaitDuration;
+
+ /** */
+ private volatile long lastCpMarkDuration;
+
+ /** */
+ private volatile long lastCpPagesWriteDuration;
+
+ /** */
+ private volatile long lastCpDuration;
+
+ /** */
+ private volatile long lastCpFsyncDuration;
+
+ /** */
+ private volatile long lastCpTotalPages;
+
+ /** */
+ private volatile long lastCpDataPages;
+
+ /** */
+ private volatile long lastCpCowPages;
+
+ /** */
+ private volatile long rateTimeInterval;
+
+ /** */
+ private volatile int subInts;
+
+ /** */
+ private volatile boolean metricsEnabled;
+
+ /** */
+ private IgniteWriteAheadLogManager wal;
+
+ /**
+ * @param metricsEnabled Metrics enabled flag.
+ * @param rateTimeInterval Rate time interval.
+ * @param subInts Number of sub-intervals.
+ */
+ public PersistenceMetricsImpl(
+ boolean metricsEnabled,
+ long rateTimeInterval,
+ int subInts
+ ) {
+ this.metricsEnabled = metricsEnabled;
+ this.rateTimeInterval = rateTimeInterval;
+ this.subInts = subInts;
+
+ resetRates();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getWalLoggingRate() {
+ if (!metricsEnabled)
+ return 0;
+
+ return ((float)walLoggingRate.getRate()) / rateTimeInterval;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getWalWritingRate() {
+ if (!metricsEnabled)
+ return 0;
+
+ return ((float)walWritingRate.getRate()) / rateTimeInterval;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getWalArchiveSegments() {
+ if (!metricsEnabled)
+ return 0;
+
+ return wal.walArchiveSegments();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getWalFsyncTimeAverage() {
+ if (!metricsEnabled)
+ return 0;
+
+ long numRate = walFsyncTimeNumber.getRate();
+
+ if (numRate == 0)
+ return 0;
+
+ return (float)walFsyncTimeDuration.getRate() / numRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointingDuration() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointLockWaitDuration() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpLockWaitDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointMarkDuration() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpMarkDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointPagesWriteDuration() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpPagesWriteDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointFsyncDuration() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpFsyncDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointTotalPagesNumber() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpTotalPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointDataPagesNumber() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpDataPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointCopiedOnWritePagesNumber() {
+ if (!metricsEnabled)
+ return 0;
+
+ return lastCpCowPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void enableMetrics() {
+ metricsEnabled = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void disableMetrics() {
+ metricsEnabled = false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rateTimeInterval(long rateTimeInterval) {
+ this.rateTimeInterval = rateTimeInterval;
+
+ resetRates();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void subIntervals(int subInts) {
+ this.subInts = subInts;
+
+ resetRates();
+ }
+
+ /**
+ * @param wal Write-ahead log manager.
+ */
+ public void wal(IgniteWriteAheadLogManager wal) {
+ this.wal = wal;
+ }
+
+ /**
+ * @return Metrics enabled flag.
+ */
+ public boolean metricsEnabled() {
+ return metricsEnabled;
+ }
+
+ /**
+ * @param lockWaitDuration Lock wait duration.
+ * @param markDuration Mark duration.
+ * @param pagesWriteDuration Pages write duration.
+ * @param fsyncDuration Total checkpoint fsync duration.
+ * @param duration Total checkpoint duration.
+ * @param totalPages Total number of all pages in checkpoint.
+ * @param dataPages Total number of data pages in checkpoint.
+ * @param cowPages Total number of COW-ed pages in checkpoint.
+ */
+ public void onCheckpoint(
+ long lockWaitDuration,
+ long markDuration,
+ long pagesWriteDuration,
+ long fsyncDuration,
+ long duration,
+ long totalPages,
+ long dataPages,
+ long cowPages
+ ) {
+ if (metricsEnabled) {
+ lastCpLockWaitDuration = lockWaitDuration;
+ lastCpMarkDuration = markDuration;
+ lastCpPagesWriteDuration = pagesWriteDuration;
+ lastCpFsyncDuration = fsyncDuration;
+ lastCpDuration = duration;
+ lastCpTotalPages = totalPages;
+ lastCpDataPages = dataPages;
+ lastCpCowPages = cowPages;
+ }
+ }
+
+ /**
+ *
+ */
+ public void onWalRecordLogged() {
+ walLoggingRate.onHit();
+ }
+
+ /**
+ * @param size Size written.
+ */
+ public void onWalBytesWritten(int size) {
+ walWritingRate.onHits(size);
+ }
+
+ /**
+ * @param nanoTime Fsync nano time.
+ */
+ public void onFsync(long nanoTime) {
+ long microseconds = nanoTime / 1_000;
+
+ walFsyncTimeDuration.onHits(microseconds);
+ walFsyncTimeNumber.onHit();
+ }
+
+ /**
+ *
+ */
+ private void resetRates() {
+ walLoggingRate = new HitRateMetrics((int)rateTimeInterval, subInts);
+ walWritingRate = new HitRateMetrics((int)rateTimeInterval, subInts);
+
+ walFsyncTimeDuration = new HitRateMetrics((int)rateTimeInterval, subInts);
+ walFsyncTimeNumber = new HitRateMetrics((int)rateTimeInterval, subInts);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsSnapshot.java
new file mode 100644
index 0000000..0de9950
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/PersistenceMetricsSnapshot.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.PersistenceMetrics;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class PersistenceMetricsSnapshot implements PersistenceMetrics {
+ /** */
+ private float walLoggingRate;
+
+ /** */
+ private float walWritingRate;
+
+ /** */
+ private int walArchiveSegments;
+
+ /** */
+ private float walFsyncTimeAvg;
+
+ /** */
+ private long lastCpDuration;
+
+ /** */
+ private long lastCpLockWaitDuration;
+
+ /** */
+ private long lastCpMmarkDuration;
+
+ /** */
+ private long lastCpPagesWriteDuration;
+
+ /** */
+ private long lastCpFsyncDuration;
+
+ /** */
+ private long lastCpTotalPages;
+
+ /** */
+ private long lastCpDataPages;
+
+ /** */
+ private long lastCpCowPages;
+
+ /**
+ * @param metrics Metrics.
+ */
+ public PersistenceMetricsSnapshot(PersistenceMetrics metrics) {
+ walLoggingRate = metrics.getWalLoggingRate();
+ walWritingRate = metrics.getWalWritingRate();
+ walArchiveSegments = metrics.getWalArchiveSegments();
+ walFsyncTimeAvg = metrics.getWalFsyncTimeAverage();
+ lastCpDuration = metrics.getLastCheckpointingDuration();
+ lastCpLockWaitDuration = metrics.getLastCheckpointLockWaitDuration();
+ lastCpMmarkDuration = metrics.getLastCheckpointMarkDuration();
+ lastCpPagesWriteDuration = metrics.getLastCheckpointPagesWriteDuration();
+ lastCpFsyncDuration = metrics.getLastCheckpointFsyncDuration();
+ lastCpTotalPages = metrics.getLastCheckpointTotalPagesNumber();
+ lastCpDataPages = metrics.getLastCheckpointDataPagesNumber();
+ lastCpCowPages = metrics.getLastCheckpointCopiedOnWritePagesNumber();
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getWalLoggingRate() {
+ return walLoggingRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getWalWritingRate() {
+ return walWritingRate;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getWalArchiveSegments() {
+ return walArchiveSegments;
+ }
+
+ /** {@inheritDoc} */
+ @Override public float getWalFsyncTimeAverage() {
+ return walFsyncTimeAvg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointingDuration() {
+ return lastCpDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointLockWaitDuration() {
+ return lastCpLockWaitDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointMarkDuration() {
+ return lastCpMmarkDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointPagesWriteDuration() {
+ return lastCpPagesWriteDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointFsyncDuration() {
+ return lastCpFsyncDuration;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointTotalPagesNumber() {
+ return lastCpTotalPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointDataPagesNumber() {
+ return lastCpDataPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long getLastCheckpointCopiedOnWritePagesNumber() {
+ return lastCpCowPages;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(PersistenceMetricsSnapshot.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RootPage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RootPage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RootPage.java
new file mode 100644
index 0000000..da9efe5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RootPage.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ *
+ */
+public class RootPage {
+ /** */
+ @GridToStringInclude
+ private FullPageId pageId;
+
+ /** */
+ private boolean allocated;
+
+ /**
+ * @param pageId Page ID.
+ * @param allocated {@code True} if was allocated new page.
+ */
+ public RootPage(final FullPageId pageId, final boolean allocated) {
+ this.pageId = pageId;
+ this.allocated = allocated;
+ }
+
+ /**
+ * @return Page ID.
+ */
+ public FullPageId pageId() {
+ return pageId;
+ }
+
+ /**
+ * @return {@code True} if was allocated.
+ */
+ public boolean isAllocated() {
+ return allocated;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(RootPage.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
new file mode 100644
index 0000000..9cc5c62
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/RowStore.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+
+/**
+ * Data store for H2 rows.
+ */
+public class RowStore {
+ /** */
+ private final FreeList freeList;
+
+ /** */
+ private final GridCacheSharedContext ctx;
+
+ /** */
+ protected final PageMemory pageMem;
+
+ /** */
+ protected final CacheObjectContext coctx;
+
+
+
+ /**
+ * @param grp Cache group.
+ * @param freeList Free list.
+ */
+ public RowStore(CacheGroupContext grp, FreeList freeList) {
+ assert grp != null;
+ assert freeList != null;
+
+ this.freeList = freeList;
+
+ ctx = grp.shared();
+ coctx = grp.cacheObjectContext();
+ pageMem = grp.memoryPolicy().pageMemory();
+ }
+
+ /**
+ * @param link Row link.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void removeRow(long link) throws IgniteCheckedException {
+ assert link != 0;
+ ctx.database().checkpointReadLock();
+
+ try {
+ freeList.removeDataRowByLink(link);
+ }
+ finally {
+ ctx.database().checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * @param row Row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRow(CacheDataRow row) throws IgniteCheckedException {
+ ctx.database().checkpointReadLock();
+
+ try {
+ freeList.insertDataRow(row);
+ }
+ finally {
+ ctx.database().checkpointReadUnlock();
+ }
+ }
+
+ /**
+ * @param link Row link.
+ * @param row New row data.
+ * @throws IgniteCheckedException If failed.
+ * @return {@code True} if was able to update row.
+ */
+ public boolean updateRow(long link, CacheDataRow row) throws IgniteCheckedException {
+ return freeList.updateDataRow(link, row);
+ }
+
+ /**
+ * @return Free list.
+ */
+ public FreeList freeList() {
+ return freeList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java
new file mode 100644
index 0000000..f5c7c8a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/FairFifoPageEvictionTracker.java
@@ -0,0 +1,76 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.evict;
+
+import java.util.LinkedList;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+
+/**
+ * On-heap FIFO page eviction tracker. Only for test purposes.
+ */
+public class FairFifoPageEvictionTracker extends PageAbstractEvictionTracker {
+ /** Page usage deque. */
+ private final LinkedList<Integer> pageUsageList = new LinkedList<>();
+
+ /**
+ * @param pageMem Page memory.
+ * @param plcCfg Memory policy configuration.
+ * @param sharedCtx Shared context.
+ */
+ public FairFifoPageEvictionTracker(
+ PageMemoryNoStoreImpl pageMem,
+ MemoryPolicyConfiguration plcCfg,
+ GridCacheSharedContext sharedCtx) {
+ super(pageMem, plcCfg, sharedCtx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void touchPage(long pageId) throws IgniteCheckedException {
+ pageUsageList.addLast(PageIdUtils.pageIndex(pageId));
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void evictDataPage() throws IgniteCheckedException {
+ evictDataPage(pageUsageList.pollFirst());
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void forgetPage(long pageId) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override protected synchronized boolean checkTouch(long pageId) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java
new file mode 100644
index 0000000..b420ecd
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/NoOpPageEvictionTracker.java
@@ -0,0 +1,50 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.evict;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+
+/**
+ *
+ */
+public class NoOpPageEvictionTracker implements PageEvictionTracker {
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void touchPage(long pageId) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void evictDataPage() throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void forgetPage(long pageId) throws IgniteCheckedException {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java
new file mode 100644
index 0000000..a524d5e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageAbstractEvictionTracker.java
@@ -0,0 +1,171 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.evict;
+
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public abstract class PageAbstractEvictionTracker implements PageEvictionTracker {
+ /** This number of least significant bits is dropped from timestamp. */
+ private static final int COMPACT_TS_SHIFT = 8; // Enough if grid works for less than 17 years.
+
+ /** Millis in day. */
+ private static final int DAY = 24 * 60 * 60 * 1000;
+
+ /** Page memory. */
+ protected final PageMemoryNoStoreImpl pageMem;
+
+ /** Tracking array size. */
+ protected final int trackingSize;
+
+ /** Base compact timestamp. */
+ private final long baseCompactTs;
+
+ /** Shared context. */
+ private final GridCacheSharedContext sharedCtx;
+
+ /**
+ * @param pageMem Page memory.
+ * @param plcCfg Memory policy configuration.
+ * @param sharedCtx Shared context.
+ */
+ PageAbstractEvictionTracker(
+ PageMemoryNoStoreImpl pageMem,
+ MemoryPolicyConfiguration plcCfg,
+ GridCacheSharedContext sharedCtx
+ ) {
+ this.pageMem = pageMem;
+
+ this.sharedCtx = sharedCtx;
+
+ trackingSize = pageMem.totalPages();
+
+ baseCompactTs = (U.currentTimeMillis() - DAY) >> COMPACT_TS_SHIFT;
+ // We subtract day to avoid fail in case of daylight shift or timezone change.
+ }
+
+ /**
+ * @param pageIdx Page index.
+ * @return true if at least one data row has been evicted
+ * @throws IgniteCheckedException If failed.
+ */
+ final boolean evictDataPage(int pageIdx) throws IgniteCheckedException {
+ long fakePageId = PageIdUtils.pageId(0, (byte)0, pageIdx);
+
+ long page = pageMem.acquirePage(0, fakePageId);
+
+ List<CacheDataRowAdapter> rowsToEvict;
+
+ try {
+ long pageAddr = pageMem.readLockForce(0, fakePageId, page);
+
+ try {
+ if (PageIO.getType(pageAddr) != PageIO.T_DATA)
+ return false; // Can't evict: page has been recycled into non-data page.
+
+ DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+
+ long realPageId = PageIO.getPageId(pageAddr);
+
+ if (!checkTouch(realPageId))
+ return false; // Can't evict: another thread concurrently invoked forgetPage()
+
+ rowsToEvict = io.forAllItems(pageAddr, new DataPageIO.CC<CacheDataRowAdapter>() {
+ @Override public CacheDataRowAdapter apply(long link) throws IgniteCheckedException {
+ CacheDataRowAdapter row = new CacheDataRowAdapter(link);
+
+ row.initFromLink(null, sharedCtx, pageMem, CacheDataRowAdapter.RowData.KEY_ONLY);
+
+ assert row.cacheId() != 0 : "Cache ID should be stored in rows of evictable cache";
+
+ return row;
+ }
+ });
+ }
+ finally {
+ pageMem.readUnlock(0, fakePageId, page);
+ }
+ }
+ finally {
+ pageMem.releasePage(0, fakePageId, page);
+ }
+
+ boolean evictionDone = false;
+
+ for (CacheDataRowAdapter dataRow : rowsToEvict) {
+ GridCacheContext<?, ?> cacheCtx = sharedCtx.cacheContext(dataRow.cacheId());
+
+ if (!cacheCtx.userCache())
+ continue;
+
+ GridCacheEntryEx entryEx = cacheCtx.isNear() ? cacheCtx.near().dht().entryEx(dataRow.key()) :
+ cacheCtx.cache().entryEx(dataRow.key());
+
+ evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true);
+ }
+
+ return evictionDone;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @return true if page was touched at least once.
+ */
+ protected abstract boolean checkTouch(long pageId);
+
+ /**
+ * @param epochMilli Time millis.
+ * @return Compact timestamp. Comparable and fits in 4 bytes.
+ */
+ final long compactTimestamp(long epochMilli) {
+ return (epochMilli >> COMPACT_TS_SHIFT) - baseCompactTs;
+ }
+
+ /**
+ * Resolves position in tracking array by page index.
+ *
+ * @param pageIdx Page index.
+ * @return Position of page in tracking array.
+ */
+ int trackingIdx(int pageIdx) {
+ return pageMem.pageSequenceNumber(pageIdx);
+ }
+
+ /**
+ * Reverse of {@link #trackingIdx(int)}.
+ *
+ * @param trackingIdx Tracking index.
+ * @return Page index.
+ */
+ int pageIdx(int trackingIdx) {
+ return pageMem.pageIndex(trackingIdx);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java
new file mode 100644
index 0000000..baa5462
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/PageEvictionTracker.java
@@ -0,0 +1,52 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.evict;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.lifecycle.LifecycleAware;
+
+/**
+ * Entry point for per-page eviction. Accepts information about touching data pages,
+ * capable of evicting "the least needed" page (according to implemented eviction algorithm).
+ */
+public interface PageEvictionTracker extends LifecycleAware {
+ /**
+ * Call this method when data page is accessed.
+ *
+ * @param pageId Page id.
+ * @throws IgniteCheckedException In case of page memory error.
+ */
+ public void touchPage(long pageId) throws IgniteCheckedException;
+
+ /**
+ * Evicts one data page.
+ * In most cases, all entries will be removed from the page.
+ * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be
+ * not possible, as some of them can be used by active transactions.
+ *
+ * @throws IgniteCheckedException In case of page memory error.
+ */
+ public void evictDataPage() throws IgniteCheckedException;
+
+ /**
+ * Call this method when last entry is removed from data page.
+ *
+ * @param pageId Page id.
+ * @throws IgniteCheckedException In case of page memory error.
+ */
+ public void forgetPage(long pageId) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java
new file mode 100644
index 0000000..00f1b16
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/Random2LruPageEvictionTracker.java
@@ -0,0 +1,180 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.evict;
+
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class Random2LruPageEvictionTracker extends PageAbstractEvictionTracker {
+ /** Evict attempts limit. */
+ private static final int EVICT_ATTEMPTS_LIMIT = 30;
+
+ /** LRU Sample size. */
+ private static final int SAMPLE_SIZE = 5;
+
+ /** Maximum sample search spin count */
+ private static final int SAMPLE_SPIN_LIMIT = SAMPLE_SIZE * 1000;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** Tracking array ptr. */
+ private long trackingArrPtr;
+
+ /**
+ * @param pageMem Page memory.
+ * @param plcCfg Policy config.
+ * @param sharedCtx Shared context.
+ */
+ public Random2LruPageEvictionTracker(
+ PageMemoryNoStoreImpl pageMem,
+ MemoryPolicyConfiguration plcCfg,
+ GridCacheSharedContext<?, ?> sharedCtx
+ ) {
+ super(pageMem, plcCfg, sharedCtx);
+
+ MemoryConfiguration memCfg = sharedCtx.kernalContext().config().getMemoryConfiguration();
+
+ assert plcCfg.getMaxSize() / memCfg.getPageSize() < Integer.MAX_VALUE;
+
+ log = sharedCtx.logger(getClass());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ trackingArrPtr = GridUnsafe.allocateMemory(trackingSize * 8);
+
+ GridUnsafe.setMemory(trackingArrPtr, trackingSize * 8, (byte)0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ GridUnsafe.freeMemory(trackingArrPtr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void touchPage(long pageId) throws IgniteCheckedException {
+ int pageIdx = PageIdUtils.pageIndex(pageId);
+
+ long latestTs = compactTimestamp(U.currentTimeMillis());
+
+ assert latestTs >= 0 && latestTs < Integer.MAX_VALUE;
+
+ boolean success;
+
+ do {
+ int trackingIdx = trackingIdx(pageIdx);
+
+ int firstTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8);
+
+ int secondTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8 + 4);
+
+ if (firstTs <= secondTs)
+ success = GridUnsafe.compareAndSwapInt(null, trackingArrPtr + trackingIdx * 8, firstTs, (int)latestTs);
+ else {
+ success = GridUnsafe.compareAndSwapInt(
+ null, trackingArrPtr + trackingIdx * 8 + 4, secondTs, (int)latestTs);
+ }
+ } while (!success);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void evictDataPage() throws IgniteCheckedException {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int evictAttemptsCnt = 0;
+
+ while (evictAttemptsCnt < EVICT_ATTEMPTS_LIMIT) {
+ int lruTrackingIdx = -1;
+
+ int lruCompactTs = Integer.MAX_VALUE;
+
+ int dataPagesCnt = 0;
+
+ int sampleSpinCnt = 0;
+
+ while (dataPagesCnt < SAMPLE_SIZE) {
+ int trackingIdx = rnd.nextInt(trackingSize);
+
+ int firstTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8);
+
+ int secondTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8 + 4);
+
+ int minTs = Math.min(firstTs, secondTs);
+
+ int maxTs = Math.max(firstTs, secondTs);
+
+ if (maxTs != 0) {
+ // We chose data page with at least one touch.
+ if (minTs < lruCompactTs) {
+ lruTrackingIdx = trackingIdx;
+
+ lruCompactTs = minTs;
+ }
+
+ dataPagesCnt++;
+ }
+
+ sampleSpinCnt++;
+
+ if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) {
+ LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT);
+
+ return;
+ }
+ }
+
+ if (evictDataPage(pageIdx(lruTrackingIdx)))
+ return;
+
+ evictAttemptsCnt++;
+ }
+
+ LT.warn(log, "Too many failed attempts to evict page: " + EVICT_ATTEMPTS_LIMIT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkTouch(long pageId) {
+ int trackingIdx = trackingIdx(PageIdUtils.pageIndex(pageId));
+
+ int firstTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8);
+
+ return firstTs != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void forgetPage(long pageId) {
+ int pageIdx = PageIdUtils.pageIndex(pageId);
+
+ int trackingIdx = trackingIdx(pageIdx);
+
+ GridUnsafe.putLongVolatile(null, trackingArrPtr + trackingIdx * 8, 0L);
+ }
+}
[06/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsNoActualWalHistoryTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsNoActualWalHistoryTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsNoActualWalHistoryTest.java
deleted file mode 100644
index 5a9da83..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsNoActualWalHistoryTest.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.file;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.util.typedef.X;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsNoActualWalHistoryTest extends GridCommonAbstractTest {
- /** Cache name. */
- private static final String CACHE_NAME = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- dbCfg.setPageSize(4 * 1024);
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalSegmentSize(4 * 1024 * 1024)
- .setWalHistorySize(2)
- .setWalSegments(10)
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration binCfg = new BinaryConfiguration();
-
- binCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(binCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalBig() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
-
- Random rnd = new Random();
-
- Map<Integer, IndexedObject> map = new HashMap<>();
-
- for (int i = 0; i < 40_000; i++) {
- if (i % 1000 == 0)
- X.println(" >> " + i);
-
- int k = rnd.nextInt(300_000);
- IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
-
- cache.put(k, v);
- map.put(k, v);
- }
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ignite.context().cache().context()
- .database();
-
- // Create many checkpoints to clean up the history.
- dbMgr.wakeupForCheckpoint("test").get();
- dbMgr.wakeupForCheckpoint("test").get();
- dbMgr.wakeupForCheckpoint("test").get();
- dbMgr.wakeupForCheckpoint("test").get();
- dbMgr.wakeupForCheckpoint("test").get();
- dbMgr.wakeupForCheckpoint("test").get();
- dbMgr.wakeupForCheckpoint("test").get();
-
- dbMgr.enableCheckpoints(false).get();
-
- for (int i = 0; i < 50; i++) {
- int k = rnd.nextInt(300_000);
- IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
-
- cache.put(k, v);
- map.put(k, v);
- }
-
- stopGrid(1);
-
- ignite = startGrid(1);
-
- cache = ignite.cache(CACHE_NAME);
-
- // Check.
- for (Integer k : map.keySet())
- assertEquals(map.get(k), cache.get(k));
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- *
- */
- private static class IndexedObject {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- private byte[] payload = new byte[1024];
-
- /**
- * @param iVal Integer value.
- */
- private IndexedObject(int iVal) {
- this.iVal = iVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof IndexedObject))
- return false;
-
- IndexedObject that = (IndexedObject)o;
-
- return iVal == that.iVal;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return iVal;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IndexedObject.class, this);
- }
- }
-
- /**
- *
- */
- private enum EnumVal {
- /** */
- VAL1,
-
- /** */
- VAL2,
-
- /** */
- VAL3
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/ignitePdsCheckpointSimulationTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/ignitePdsCheckpointSimulationTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/ignitePdsCheckpointSimulationTest.java
deleted file mode 100644
index b30162d..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/ignitePdsCheckpointSimulationTest.java
+++ /dev/null
@@ -1,1000 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.file;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.nio.ByteOrder;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.TrackingPageIO;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
-import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheOperation;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class ignitePdsCheckpointSimulationTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final int TOTAL_PAGES = 1000;
-
- /** */
- private static final boolean VERBOSE = false;
-
- /** Cache name. */
- private final String cacheName = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration ccfg = new CacheConfiguration(cacheName);
-
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
-
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setCheckpointingFrequency(500)
- .setWalMode(WALMode.LOG_ONLY)
- .setAlwaysWriteFullPages(true)
- );
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testCheckpointSimulationMultiThreaded() throws Exception {
- IgniteEx ig = startGrid(0);
-
- GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
-
- IgnitePageStoreManager pageStore = shared.pageStore();
-
- U.sleep(1_000);
-
- // Disable integrated checkpoint thread.
- dbMgr.enableCheckpoints(false).get();
-
- // Must put something in partition 0 in order to initialize meta page.
- // Otherwise we will violate page store integrity rules.
- ig.cache(cacheName).put(0, 0);
-
- PageMemory mem = shared.database().memoryPolicy(null).pageMemory();
-
- IgniteBiTuple<Map<FullPageId, Integer>, WALPointer> res;
-
- try {
- res = runCheckpointing(ig, (PageMemoryImpl)mem, pageStore, shared.wal(),
- shared.cache().cache(cacheName).context().cacheId());
- }
- catch (Throwable th) {
- log().error("Error while running checkpointing", th);
-
- throw th;
- }
- finally {
- dbMgr.enableCheckpoints(true).get();
-
- stopAllGrids(false);
- }
-
- ig = startGrid(0);
-
- shared = ig.context().cache().context();
-
- dbMgr = (GridCacheDatabaseSharedManager)shared.database();
-
- dbMgr.enableCheckpoints(false).get();
-
- mem = shared.database().memoryPolicy(null).pageMemory();
-
- verifyReads(res.get1(), mem, res.get2(), shared.wal());
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testGetForInitialWrite() throws Exception {
- IgniteEx ig = startGrid(0);
-
- GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
-
- int cacheId = shared.cache().cache(cacheName).context().cacheId();
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
-
- // Disable integrated checkpoint thread.
- dbMgr.enableCheckpoints(false);
-
- PageMemory mem = shared.database().memoryPolicy(null).pageMemory();
-
- IgniteWriteAheadLogManager wal = shared.wal();
-
- WALPointer start = wal.log(new CheckpointRecord(null, false));
-
- final FullPageId[] initWrites = new FullPageId[10];
-
- ig.context().cache().context().database().checkpointReadLock();
-
- try {
- for (int i = 0; i < initWrites.length; i++)
- initWrites[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
-
- // Check getForInitialWrite methods.
- for (FullPageId fullId : initWrites) {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
- try {
- long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- try {
- DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
-
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
- PageUtils.putByte(pageAddr, i, (byte)0xAB);
- }
- finally {
- mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
- }
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
-
- wal.fsync(null);
- }
- finally {
- ig.context().cache().context().database().checkpointReadUnlock();
- stopAllGrids(false);
- }
-
- ig = startGrid(0);
-
- shared = ig.context().cache().context();
-
- dbMgr = (GridCacheDatabaseSharedManager)shared.database();
-
- dbMgr.enableCheckpoints(false);
-
- wal = shared.wal();
-
- try (WALIterator it = wal.replay(start)) {
- it.nextX();
-
- for (FullPageId initialWrite : initWrites) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
- assertTrue(String.valueOf(tup.get2()), tup.get2() instanceof PageSnapshot);
-
- PageSnapshot snap = (PageSnapshot)tup.get2();
-
- FullPageId actual = snap.fullPageId();
-
- //there are extra tracking pages, skip them
- if (TrackingPageIO.VERSIONS.latest().trackingPageFor(actual.pageId(), mem.pageSize()) == actual.pageId()) {
- tup = it.nextX();
-
- assertTrue(tup.get2() instanceof PageSnapshot);
-
- actual = ((PageSnapshot)tup.get2()).fullPageId();
- }
-
- assertEquals(initialWrite, actual);
- }
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testDataWalEntries() throws Exception {
- IgniteEx ig = startGrid(0);
-
- GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
- GridCacheContext<Object, Object> cctx = sharedCtx.cache().cache(cacheName).context();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
- IgniteWriteAheadLogManager wal = sharedCtx.wal();
-
- assertTrue(wal.isAlwaysWriteFullPages());
-
- db.enableCheckpoints(false);
-
- final int cnt = 10;
-
- List<DataEntry> entries = new ArrayList<>(cnt);
-
- for (int i = 0; i < cnt; i++) {
- GridCacheOperation op = i % 2 == 0 ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE;
-
- KeyCacheObject key = cctx.toCacheKeyObject(i);
-
- CacheObject val = null;
-
- if (op != GridCacheOperation.DELETE)
- val = cctx.toCacheObject("value-" + i);
-
- entries.add(new DataEntry(cctx.cacheId(), key, val, op, null, cctx.versions().next(), 0L,
- cctx.affinity().partition(i), i));
- }
-
- UUID cpId = UUID.randomUUID();
-
- WALPointer start = wal.log(new CheckpointRecord(cpId, null, false));
-
- wal.fsync(start);
-
- for (DataEntry entry : entries)
- wal.log(new DataRecord(entry));
-
- WALPointer end = wal.log(new CheckpointRecord(cpId, start, true));
-
- wal.fsync(end);
-
- // Data will not be written to the page store.
- stopAllGrids();
-
- ig = startGrid(0);
-
- sharedCtx = ig.context().cache().context();
- cctx = sharedCtx.cache().cache(cacheName).context();
-
- db = (GridCacheDatabaseSharedManager)sharedCtx.database();
- wal = sharedCtx.wal();
-
- db.enableCheckpoints(false);
-
- try (WALIterator it = wal.replay(start)) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
- assert tup.get2() instanceof CheckpointRecord;
-
- assertEquals(start, tup.get1());
-
- CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
-
- assertEquals(cpId, cpRec.checkpointId());
- assertNull(cpRec.checkpointMark());
- assertFalse(cpRec.end());
-
- int idx = 0;
- CacheObjectContext coctx = cctx.cacheObjectContext();
-
- while (idx < entries.size()) {
- tup = it.nextX();
-
- assert tup.get2() instanceof DataRecord;
-
- DataRecord dataRec = (DataRecord)tup.get2();
-
- DataEntry entry = entries.get(idx);
-
- assertEquals(1, dataRec.writeEntries().size());
-
- DataEntry readEntry = dataRec.writeEntries().get(0);
-
- assertEquals(entry.cacheId(), readEntry.cacheId());
- assertEquals(entry.key().<Integer>value(coctx, true), readEntry.key().<Integer>value(coctx, true));
- assertEquals(entry.op(), readEntry.op());
-
- if (entry.op() == GridCacheOperation.UPDATE)
- assertEquals(entry.value().value(coctx, true), readEntry.value().value(coctx, true));
- else
- assertNull(entry.value());
-
- assertEquals(entry.writeVersion(), readEntry.writeVersion());
- assertEquals(entry.nearXidVersion(), readEntry.nearXidVersion());
- assertEquals(entry.partitionCounter(), readEntry.partitionCounter());
-
- idx++;
- }
-
- tup = it.nextX();
-
- assert tup.get2() instanceof CheckpointRecord;
-
- assertEquals(end, tup.get1());
-
- cpRec = (CheckpointRecord)tup.get2();
-
- assertEquals(cpId, cpRec.checkpointId());
- assertEquals(start, cpRec.checkpointMark());
- assertTrue(cpRec.end());
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageWalEntries() throws Exception {
- IgniteEx ig = startGrid(0);
-
- GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
- int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
- PageMemory pageMem = sharedCtx.database().memoryPolicy(null).pageMemory();
- IgniteWriteAheadLogManager wal = sharedCtx.wal();
-
- db.enableCheckpoints(false).get();
-
- int pageCnt = 100;
-
- List<FullPageId> pageIds = new ArrayList<>();
-
- for (int i = 0; i < pageCnt; i++) {
- db.checkpointReadLock();
- try {
- pageIds.add(new FullPageId(pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION,
- PageIdAllocator.FLAG_IDX), cacheId));
- }
- finally {
- db.checkpointReadUnlock();
- }
- }
-
- UUID cpId = UUID.randomUUID();
-
- WALPointer start = wal.log(new CheckpointRecord(cpId, null, false));
-
- wal.fsync(start);
-
- ig.context().cache().context().database().checkpointReadLock();
-
- try {
- for (FullPageId pageId : pageIds)
- writePageData(pageId, pageMem);
- }
- finally {
- ig.context().cache().context().database().checkpointReadUnlock();
- }
-
- WALPointer end = wal.log(new CheckpointRecord(cpId, start, true));
-
- wal.fsync(end);
-
- // Data will not be written to the page store.
- stopAllGrids();
-
- ig = startGrid(0);
-
- sharedCtx = ig.context().cache().context();
-
- db = (GridCacheDatabaseSharedManager)sharedCtx.database();
- wal = sharedCtx.wal();
-
- db.enableCheckpoints(false);
-
- try (WALIterator it = wal.replay(start)) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
- assert tup.get2() instanceof CheckpointRecord : tup.get2();
-
- assertEquals(start, tup.get1());
-
- CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
-
- assertEquals(cpId, cpRec.checkpointId());
- assertNull(cpRec.checkpointMark());
- assertFalse(cpRec.end());
-
- int idx = 0;
-
- while (idx < pageIds.size()) {
- tup = it.nextX();
-
- assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
-
- PageSnapshot snap = (PageSnapshot)tup.get2();
-
- //there are extra tracking pages, skip them
- if (TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize()) == snap.fullPageId().pageId()) {
- tup = it.nextX();
-
- assertTrue(tup.get2() instanceof PageSnapshot);
-
- snap = (PageSnapshot)tup.get2();
- }
-
- assertEquals(pageIds.get(idx), snap.fullPageId());
-
- idx++;
- }
-
- tup = it.nextX();
-
- assert tup.get2() instanceof CheckpointRecord;
-
- assertEquals(end, tup.get1());
-
- cpRec = (CheckpointRecord)tup.get2();
-
- assertEquals(cpId, cpRec.checkpointId());
- assertEquals(start, cpRec.checkpointMark());
- assertTrue(cpRec.end());
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testDirtyFlag() throws Exception {
- IgniteEx ig = startGrid(0);
-
- GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
-
- int cacheId = shared.cache().cache(cacheName).context().cacheId();
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
-
- // Disable integrated checkpoint thread.
- dbMgr.enableCheckpoints(false);
-
- PageMemoryEx mem = (PageMemoryEx) dbMgr.memoryPolicy(null).pageMemory();
-
- ig.context().cache().context().database().checkpointReadLock();
-
- FullPageId[] pageIds = new FullPageId[100];
-
- try {
- for (int i = 0; i < pageIds.length; i++)
- pageIds[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
-
- for (FullPageId fullId : pageIds) {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); //page is dirty right after allocation
-
- long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- PageIO.setPageId(pageAddr, fullId.pageId());
-
- try {
- assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
- }
- finally {
- mem.writeUnlock(fullId.cacheId(), fullId.pageId(),page, null,true);
- }
-
- assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- }
- finally {
- ig.context().cache().context().database().checkpointReadUnlock();
- }
-
- Collection<FullPageId> cpPages = mem.beginCheckpoint();
-
- ig.context().cache().context().database().checkpointReadLock();
-
- try {
- for (FullPageId fullId : pageIds) {
- assertTrue(cpPages.contains(fullId));
-
- ByteBuffer buf = ByteBuffer.allocate(mem.pageSize());
-
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
-
- long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- try {
- assertFalse(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
-
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
- PageUtils.putByte(pageAddr, i, (byte)1);
- }
- finally {
- mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
- }
-
- assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
-
- buf.rewind();
-
- mem.getForCheckpoint(fullId, buf, null);
-
- buf.position(PageIO.COMMON_HEADER_END);
-
- while (buf.hasRemaining())
- assertEquals((byte)0, buf.get());
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- }
- finally {
- ig.context().cache().context().database().checkpointReadUnlock();
- }
-
- mem.finishCheckpoint();
-
- for (FullPageId fullId : pageIds) {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
- try {
- assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- private void writePageData(FullPageId fullId, PageMemory mem) throws Exception {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
- try {
- long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- try {
- DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
- PageUtils.putByte(pageAddr, i, (byte)rnd.nextInt(255));
- }
- finally {
- mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
- }
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
-
- /**
- * @param res Result map to verify.
- * @param mem Memory.
- */
- private void verifyReads(
- Map<FullPageId, Integer> res,
- PageMemory mem,
- WALPointer start,
- IgniteWriteAheadLogManager wal
- ) throws Exception {
- Map<FullPageId, byte[]> replay = new HashMap<>();
-
- try (WALIterator it = wal.replay(start)) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
- assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
-
- CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
-
- while (it.hasNextX()) {
- tup = it.nextX();
-
- WALRecord rec = tup.get2();
-
- if (rec instanceof CheckpointRecord) {
- CheckpointRecord end = (CheckpointRecord)rec;
-
- // Found the finish mark.
- if (end.checkpointId().equals(cpRec.checkpointId()) && end.end())
- break;
- }
- else if (rec instanceof PageSnapshot) {
- PageSnapshot page = (PageSnapshot)rec;
-
- replay.put(page.fullPageId(), page.pageData());
- }
- }
- }
-
- // Check read-through from the file store.
- for (Map.Entry<FullPageId, Integer> entry : res.entrySet()) {
- FullPageId fullId = entry.getKey();
- int state = entry.getValue();
-
- if (state == -1) {
- info("Page was never written: " + fullId);
-
- continue;
- }
-
- byte[] walData = replay.get(fullId);
-
- assertNotNull("Missing WAL record for a written page: " + fullId, walData);
-
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
- try {
- long pageAddr = mem.readLock(fullId.cacheId(), fullId.pageId(), page);
-
- try {
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
- assertEquals("Invalid state [pageId=" + fullId + ", pos=" + i + ']',
- state & 0xFF, PageUtils.getByte(pageAddr, i) & 0xFF);
-
- assertEquals("Invalid WAL state [pageId=" + fullId + ", pos=" + i + ']',
- state & 0xFF, walData[i] & 0xFF);
- }
- }
- finally {
- mem.readUnlock(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- }
-
- /**
- * @param mem Memory to use.
- * @param storeMgr Store manager.
- * @param cacheId Cache ID.
- * @return Result map of random operations.
- * @throws Exception If failure occurred.
- */
- private IgniteBiTuple<Map<FullPageId, Integer>, WALPointer> runCheckpointing(
- final IgniteEx ig,
- final PageMemoryImpl mem,
- final IgnitePageStoreManager storeMgr,
- final IgniteWriteAheadLogManager wal,
- final int cacheId
- ) throws Exception {
- final ConcurrentMap<FullPageId, Integer> resMap = new ConcurrentHashMap<>();
-
- final FullPageId pages[] = new FullPageId[TOTAL_PAGES];
-
- Set<FullPageId> allocated = new HashSet<>();
-
- for (int i = 0; i < TOTAL_PAGES; i++) {
- FullPageId fullId = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
-
- resMap.put(fullId, -1);
-
- pages[i] = fullId;
-
- allocated.add(fullId);
- }
-
- final AtomicBoolean run = new AtomicBoolean(true);
-
- // Simulate transaction lock.
- final ReadWriteLock updLock = new ReentrantReadWriteLock();
-
- // Mark the start position.
- CheckpointRecord cpRec = new CheckpointRecord(null, false);
-
- WALPointer start = wal.log(cpRec);
-
- wal.fsync(start);
-
- IgniteInternalFuture<Long> updFut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- while (true) {
- FullPageId fullId = pages[ThreadLocalRandom.current().nextInt(TOTAL_PAGES)];
-
- updLock.readLock().lock();
-
- try {
- if (!run.get())
- return null;
-
- ig.context().cache().context().database().checkpointReadLock();
-
- try {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- PageIO.setPageId(pageAddr, fullId.pageId());
-
- try {
- int state = resMap.get(fullId);
-
- if (state != -1) {
- if (VERBOSE)
- info("Verify page [fullId=" + fullId + ", state=" + state +
- ", buf=" + pageAddr +
- ", bhc=" + U.hexLong(System.identityHashCode(pageAddr)) +
- ", page=" + U.hexLong(System.identityHashCode(page)) + ']');
-
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
- assertEquals("Verify page failed [fullId=" + fullId +
- ", i=" + i +
- ", state=" + state +
- ", buf=" + pageAddr +
- ", bhc=" + U.hexLong(System.identityHashCode(pageAddr)) +
- ", page=" + U.hexLong(System.identityHashCode(page)) + ']',
- state & 0xFF, PageUtils.getByte(pageAddr, i) & 0xFF);
- }
-
- state = (state + 1) & 0xFF;
-
- if (VERBOSE)
- info("Write page [fullId=" + fullId + ", state=" + state +
- ", buf=" + pageAddr +
- ", bhc=" + U.hexLong(System.identityHashCode(pageAddr)) +
- ", page=" + U.hexLong(System.identityHashCode(page)) + ']');
-
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
- PageUtils.putByte(pageAddr, i, (byte)state);
-
- resMap.put(fullId, state);
- }
- finally {
- mem.writeUnlock(fullId.cacheId(), fullId.pageId(),page, null,true);
- }
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(),page);}
- }
- finally {
- ig.context().cache().context().database().checkpointReadUnlock();
- }
- }
- finally {
- updLock.readLock().unlock();
- }
- }
- }
- }, 8, "update-thread");
-
- int checkpoints = 20;
-
- while (checkpoints > 0) {
- Map<FullPageId, Integer> snapshot = null;
-
- Collection<FullPageId> pageIds;
-
- updLock.writeLock().lock();
-
- try {
- snapshot = new HashMap<>(resMap);
-
- pageIds = mem.beginCheckpoint();
-
- checkpoints--;
-
- if (checkpoints == 0)
- // No more writes should be done at this point.
- run.set(false);
-
- info("Acquired pages for checkpoint: " + pageIds.size());
- }
- finally {
- updLock.writeLock().unlock();
- }
-
- boolean ok = false;
-
- try {
- ByteBuffer tmpBuf = ByteBuffer.allocate(mem.pageSize());
-
- tmpBuf.order(ByteOrder.nativeOrder());
-
- long begin = System.currentTimeMillis();
-
- long cp = 0;
-
- long write = 0;
-
- for (FullPageId fullId : pageIds) {
- long cpStart = System.nanoTime();
-
- Integer tag = mem.getForCheckpoint(fullId, tmpBuf, null);
-
- if (tag == null)
- continue;
-
- long cpEnd = System.nanoTime();
-
- cp += cpEnd - cpStart;
-
- Integer state = snapshot.get(fullId);
-
- if (allocated.contains(fullId) && state != -1) {
- tmpBuf.rewind();
-
- Integer first = null;
-
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
- int val = tmpBuf.get(i) & 0xFF;
-
- if (first == null)
- first = val;
-
- // Avoid string concat.
- if (first != val)
- assertEquals("Corrupted buffer at position [pageId=" + fullId + ", pos=" + i + ']',
- (int)first, val);
-
- // Avoid string concat.
- if (state != val)
- assertEquals("Invalid value at position [pageId=" + fullId + ", pos=" + i + ']',
- (int)state, val);
- }
- }
-
- tmpBuf.rewind();
-
- long writeStart = System.nanoTime();
-
- storeMgr.write(cacheId, fullId.pageId(), tmpBuf, tag);
-
- long writeEnd = System.nanoTime();
-
- write += writeEnd - writeStart;
-
- tmpBuf.rewind();
- }
-
- long syncStart = System.currentTimeMillis();
-
- storeMgr.sync(cacheId, 0);
-
- long end = System.currentTimeMillis();
-
- info("Written pages in " + (end - begin) + "ms, copy took " + (cp / 1_000_000) + "ms, " +
- "write took " + (write / 1_000_000) + "ms, sync took " + (end - syncStart) + "ms");
-
- ok = true;
- }
- finally {
- info("Finishing checkpoint...");
-
- mem.finishCheckpoint();
-
- info("Finished checkpoint");
-
- if (!ok) {
- info("Cancelling updates...");
-
- run.set(false);
-
- updFut.get();
- }
- }
-
- if (checkpoints != 0)
- Thread.sleep(2_000);
- }
-
- info("checkpoints=" + checkpoints + ", done=" + updFut.isDone());
-
- updFut.get();
-
- // Mark the end.
- wal.fsync(wal.log(new CheckpointRecord(cpRec.checkpointId(), start, true)));
-
- assertEquals(0, mem.activePagesCount());
-
- for (FullPageId fullId : pages) {
-
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" +
- fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page));
-
- assertFalse("Page is dirty after the last checkpoint: [cacheId=" +
- fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
-
- return F.t((Map<FullPageId, Integer>)resMap, start);
- }
-
- /**
- *
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgnitePdsWalTlbTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgnitePdsWalTlbTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgnitePdsWalTlbTest.java
deleted file mode 100644
index 03b1384..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgnitePdsWalTlbTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal;
-
-import javax.cache.CacheException;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.configuration.PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE;
-
-/**
- *
- */
-public class IgnitePdsWalTlbTest extends GridCommonAbstractTest {
- /** Ip finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Cache name. */
- private static final String CACHE_NAME = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>(CACHE_NAME);
-
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setCheckpointingPageBufferSize(DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE + 1)
- .setTlbSize(640000000)
- );
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- if (gridName.endsWith("1"))
- cfg.setClientMode(true);
-
- cfg.setDiscoverySpi(discoSpi);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 30_000;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- stopAllGrids();
-
- startGrids(2);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalDirectOutOfMemory() throws Exception {
- IgniteEx ig = grid(1);
-
- boolean locked = true;
-
- try {
- IgniteDataStreamer<Integer, Integer> streamer = ig.dataStreamer(CACHE_NAME);
-
- for (int i = 0; i < 100_000; i++) {
- streamer.addData(i, 1);
-
- if (i > 0 && i % 10_000 == 0)
- info("Done put: " + i);
- }
- }
- catch (CacheException ignore) {
- // expected
- locked = false;
- }
- finally {
- assertFalse(locked);
-
- stopAllGrids();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalDirectoriesConfigurationTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalDirectoriesConfigurationTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalDirectoriesConfigurationTest.java
deleted file mode 100644
index 4cca401..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalDirectoriesConfigurationTest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal;
-
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgniteWalDirectoriesConfigurationTest extends GridCommonAbstractTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
-
- pCfg.setWalStorePath("test/db/wal");
-
- cfg.setPersistentStoreConfiguration(pCfg);
-
- return cfg;
- }
-
- /**
- *
- */
- public void testPartialWalConfigurationNotAllowed() {
- try {
- startGrid();
- }
- catch (Exception ignore) {
- return;
- }
-
- fail("Node successfully started with incorrect configuration, exception is expected.");
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalHistoryReservationsTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalHistoryReservationsTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalHistoryReservationsTest.java
deleted file mode 100644
index fc46c2e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalHistoryReservationsTest.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal;
-
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.wal.FileWriteAheadLogManager;
-import org.apache.ignite.internal.util.lang.GridAbsPredicate;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
-
-/**
- *
- */
-public class IgniteWalHistoryReservationsTest extends GridCommonAbstractTest {
- /** */
- private volatile boolean client;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setClientMode(client);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- memCfg.setMemoryPolicies(new MemoryPolicyConfiguration().setInitialSize(200 * 1024 * 1024)
- .setMaxSize(200 * 1024 * 1024).setName("dfltMemPlc"));
-
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- CacheConfiguration ccfg1 = new CacheConfiguration();
-
- ccfg1.setName("cache1");
- ccfg1.setBackups(1);
- ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- cfg.setCacheConfiguration(ccfg1);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- System.clearProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
-
- client = false;
-
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testReservedOnExchange() throws Exception {
- System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
-
- final int entryCnt = 10_000;
- final int initGridCnt = 4;
-
- final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt);
-
- IgniteCache<Object, Object> cache = ig0.cache("cache1");
-
- for (int k = 0; k < entryCnt; k++)
- cache.put(k, k);
-
- forceCheckpoint();
-
- for (int k = 0; k < entryCnt; k++)
- cache.put(k, k * 2);
-
- forceCheckpoint();
-
- for (int k = 0; k < entryCnt; k++)
- cache.put(k, k);
-
- forceCheckpoint();
-
- Lock lock = cache.lock(0);
-
- lock.lock();
-
- try {
- GridTestUtils.runAsync(new Runnable() {
- @Override public void run() {
- try {
- startGrid(initGridCnt);
- }
- catch (Exception e) {
- fail(e.getMessage());
- }
- }
- });
-
- boolean reserved = GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- for (int g = 0; g < initGridCnt; g++) {
- IgniteEx ig = grid(g);
-
- FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
- Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
-
- synchronized (archiver) {
- Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
-
- if (reserved.isEmpty())
- return false;
- }
- }
-
- return true;
- }
- }, 10_000);
-
- assert reserved;
- }
- finally {
- lock.unlock();
- }
-
- boolean released = GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- for (int g = 0; g < initGridCnt; g++) {
- IgniteEx ig = grid(g);
-
- FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
- Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
-
- synchronized (archiver) {
- Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
-
- if (!reserved.isEmpty())
- return false;
- }
- }
-
- return true;
- }
- }, 10_000);
-
- assert released;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
- System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
-
- int entryCnt = 10_000;
-
- Ignite ig0 = startGrids(2);
-
- IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
-
- for (int k = 0; k < entryCnt; k++)
- cache.put(k, k);
-
- forceCheckpoint();
-
- stopGrid(1);
-
- for (int k = 0; k < entryCnt; k += 2)
- cache.remove(k);
-
- forceCheckpoint();
-
- Ignite ig1 = startGrid(1);
-
- IgniteCache<Integer, Integer> cache1 = ig1.cache("cache1");
-
- assertEquals(entryCnt / 2, cache.size());
- assertEquals(entryCnt / 2, cache1.size());
-
- for (Integer k = 0; k < entryCnt; k++) {
- if (k % 2 == 0) {
- assertTrue("k=" + k, !cache.containsKey(k));
- assertTrue("k=" + k, !cache1.containsKey(k));
- }
- else {
- assertEquals("k=" + k, k, cache.get(k));
- assertEquals("k=" + k, k, cache1.get(k));
- }
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
- int entryCnt = 10_000;
-
- Ignite ig0 = startGrids(2);
-
- IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
-
- for (int k = 0; k < entryCnt; k++)
- cache.put(k, k);
-
- forceCheckpoint();
-
- stopGrid(1);
-
- for (int k = 0; k < entryCnt; k += 2)
- cache.remove(k);
-
- forceCheckpoint();
-
- for (Integer k = 0; k < entryCnt; k++) {
- if (k % 2 == 0)
- assertTrue("k=" + k, !cache.containsKey(k));
- else
- assertEquals("k=" + k, k, cache.get(k));
- }
-
- Ignite ig1 = startGrid(1);
-
- IgniteCache<Integer, Integer> cache1 = ig1.cache("cache1");
-
- assertEquals(entryCnt / 2, cache.size());
- assertEquals(entryCnt / 2, cache1.size());
-
- for (Integer k = 0; k < entryCnt; k++) {
- if (k % 2 == 0) {
- assertTrue("k=" + k, !cache.containsKey(k));
- assertTrue("k=" + k, !cache1.containsKey(k));
- }
- else {
- assertEquals("k=" + k, k, cache.get(k));
- assertEquals("k=" + k, k, cache1.get(k));
- }
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testNodeLeftDuringExchange() throws Exception {
- System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
-
- final int entryCnt = 10_000;
- final int initGridCnt = 4;
-
- final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt);
-
- IgniteCache<Object, Object> cache = ig0.cache("cache1");
-
- for (int k = 0; k < entryCnt; k++)
- cache.put(k, k);
-
- forceCheckpoint();
-
- Lock lock = cache.lock(0);
-
- lock.lock();
-
- try {
- GridTestUtils.runAsync(new Runnable() {
- @Override public void run() {
- try {
- startGrid(initGridCnt);
- }
- catch (Exception e) {
- fail(e.getMessage());
- }
- }
- });
-
- boolean reserved = GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- for (int g = 0; g < initGridCnt; g++) {
- IgniteEx ig = grid(g);
-
- FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
- Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
-
- synchronized (archiver) {
- Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
-
- if (reserved.isEmpty())
- return false;
- }
- }
-
- return true;
- }
- }, 10_000);
-
- assert reserved;
-
- stopGrid(Integer.toString(initGridCnt - 1), true, false);
- }
- finally {
- lock.unlock();
- }
-
- boolean released = GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- for (int g = 0; g < initGridCnt - 1; g++) {
- IgniteEx ig = grid(g);
-
- FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
-
- Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
-
- synchronized (archiver) {
- Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
-
- if (!reserved.isEmpty())
- return false;
- }
- }
-
- return true;
- }
- }, 10_000);
-
- assert released;
-
- awaitPartitionMapExchange();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- *
- */
- private void forceCheckpoint() throws Exception {
- for (Ignite ignite : G.allGrids()) {
- if (ignite.cluster().localNode().isClient())
- continue;
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
- .cache().context().database();
-
- dbMgr.waitForCheckpoint("test");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoverySeveralRestartsTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoverySeveralRestartsTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoverySeveralRestartsTest.java
deleted file mode 100644
index d70bea2..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/wal/IgniteWalRecoverySeveralRestartsTest.java
+++ /dev/null
@@ -1,347 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.wal;
-
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgniteWalRecoverySeveralRestartsTest extends GridCommonAbstractTest {
- /** */
- public static final int PAGE_SIZE = 1024;
-
- /** */
- private static final int KEYS_COUNT = 100_000;
-
- /** */
- private static final int LARGE_KEYS_COUNT = 5_000;
-
- /** */
- private static final Random rnd = new Random(System.currentTimeMillis());
-
- /** */
- private String cacheName = "test";
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 3600_000;
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(cacheName);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
- ccfg.setIndexedTypes(Integer.class, IndexedObject.class);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
- ccfg.setReadFromBackup(true);
-
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- dbCfg.setPageSize(PAGE_SIZE);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(500 * 1024 * 1024);
- memPlcCfg.setMaxSize(500 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration binCfg = new BinaryConfiguration();
-
- binCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(binCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRecoverySeveralRestarts() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- Random locRandom = ThreadLocalRandom.current();
-
- try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer(cacheName)) {
- for (int i = 0; i < KEYS_COUNT; ++i) {
- if (i % (KEYS_COUNT / 100) == 0)
- info("Loading " + i * 100 / KEYS_COUNT + "%");
-
- dataLdr.addData(i, new IndexedObject(i));
- }
- }
-
- int size = ignite.cache(cacheName).size();
-
- for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
- stopGrid(1, true);
-
- info("Restart #" + restartCnt);
- U.sleep(500);
-
- ignite = startGrid(1);
-
- IgniteCache<Integer, IndexedObject> cache = ignite.cache(cacheName);
-
- assertEquals(size, cache.size());
-
- info("Restart #" + restartCnt);
-
- for (int i = 0; i < KEYS_COUNT / 100; ++i) {
- assertNotNull(cache.get(locRandom.nextInt(KEYS_COUNT / 100)));
-
- cache.put(locRandom.nextInt(KEYS_COUNT / 100), new IndexedObject(locRandom.nextInt(KEYS_COUNT / 100)));
- }
-
- cache.put(KEYS_COUNT + restartCnt, new IndexedObject(KEYS_COUNT + restartCnt));
-
- // Check recovery for partition meta pages.
- size = cache.size();
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRecoveryWithDynamicCache() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- CacheConfiguration<Integer, IndexedObject> dynCacheCfg = new CacheConfiguration<>();
-
- dynCacheCfg.setName("dyncache");
- dynCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- dynCacheCfg.setRebalanceMode(CacheRebalanceMode.NONE);
- dynCacheCfg.setIndexedTypes(Integer.class, IndexedObject.class);
- dynCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dynCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
- dynCacheCfg.setReadFromBackup(true);
-
- ignite.getOrCreateCache(dynCacheCfg);
-
- try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer("dyncache")) {
- for (int i = 0; i < KEYS_COUNT; ++i) {
- if (i % (KEYS_COUNT / 100) == 0)
- info("Loading " + i * 100 / KEYS_COUNT + "%");
-
- dataLdr.addData(i, new IndexedObject(i));
- }
- }
-
- for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
- stopGrid(1, true);
-
- info("Restart #" + restartCnt);
- U.sleep(500);
-
- ignite = startGrid(1);
-
- ThreadLocalRandom locRandom = ThreadLocalRandom.current();
-
- IgniteCache<Integer, IndexedObject> cache = ignite.getOrCreateCache(dynCacheCfg);
-
- for (int i = 0; i < KEYS_COUNT; ++i)
- assertNotNull(cache.get(locRandom.nextInt(KEYS_COUNT)));
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testWalRecoveryWithDynamicCacheLargeObjects() throws Exception {
- try {
- IgniteEx ignite = startGrid(1);
-
- CacheConfiguration<Integer, IndexedObject> dynCacheCfg = new CacheConfiguration<>();
-
- dynCacheCfg.setName("dyncache");
- dynCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- dynCacheCfg.setRebalanceMode(CacheRebalanceMode.NONE);
- dynCacheCfg.setIndexedTypes(Integer.class, IndexedObject.class);
- dynCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dynCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
- dynCacheCfg.setReadFromBackup(true);
-
- ignite.getOrCreateCache(dynCacheCfg);
-
- try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer("dyncache")) {
- for (int i = 0; i < LARGE_KEYS_COUNT; ++i) {
- if (i % (LARGE_KEYS_COUNT / 100) == 0)
- info("Loading " + i * 100 / LARGE_KEYS_COUNT + "%");
-
- IndexedObject obj = new IndexedObject(i);
-
- obj.payload = new byte[PAGE_SIZE + 2];
-
- dataLdr.addData(i, obj);
- }
- }
-
- for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
- stopGrid(1, true);
-
- info("Restart #" + restartCnt);
- U.sleep(500);
-
- ignite = startGrid(1);
-
- ThreadLocalRandom locRandom = ThreadLocalRandom.current();
-
- IgniteCache<Integer, IndexedObject> cache = ignite.getOrCreateCache(dynCacheCfg);
-
- for (int i = 0; i < LARGE_KEYS_COUNT; ++i) {
- IndexedObject val = cache.get(locRandom.nextInt(LARGE_KEYS_COUNT));
-
- assertNotNull(val);
-
- assertEquals(PAGE_SIZE + 2, val.payload.length);
- }
- }
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- *
- */
- private static class IndexedObject {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String strVal0;
-
- /** */
- @QuerySqlField(index = true)
- private String strVal1;
-
- /** */
- private byte[] payload;
-
- /**
- * @param iVal Integer value.
- */
- private IndexedObject(int iVal) {
- this.iVal = iVal;
-
- strVal0 = "String value #0 " + iVal + " " + GridTestUtils.randomString(rnd, 256);
- strVal1 = GridTestUtils.randomString(rnd, 256);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- IndexedObject obj = (IndexedObject)o;
-
- if (iVal != obj.iVal)
- return false;
-
- if (strVal0 != null ? !strVal0.equals(obj.strVal0) : obj.strVal0 != null)
- return false;
-
- return strVal1 != null ? strVal1.equals(obj.strVal1) : obj.strVal1 == null;
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (strVal0 != null ? strVal0.hashCode() : 0);
- res = 31 * res + (strVal1 != null ? strVal1.hashCode() : 0);
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(IndexedObject.class, this);
- }
- }
-}
[46/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRow.java
deleted file mode 100644
index cc26b21..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRow.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-
-/**
- * Cache data row.
- */
-public interface CacheDataRow extends CacheSearchRow {
- /**
- * @return Cache value.
- */
- public CacheObject value();
-
- /**
- * @return Cache entry version.
- */
- public GridCacheVersion version();
-
- /**
- * @return Expire time.
- */
- public long expireTime();
-
- /**
- * @return Partition for this key.
- */
- public int partition();
-
- /**
- * @param link Link for this row.
- */
- public void link(long link);
-
- /**
- * @param key Key.
- */
- public void key(KeyCacheObject key);
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRowAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRowAdapter.java
deleted file mode 100644
index a25d794..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheDataRowAdapter.java
+++ /dev/null
@@ -1,593 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.IncompleteCacheObject;
-import org.apache.ignite.internal.processors.cache.IncompleteObject;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.tree.io.CacheVersionIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPagePayload;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.Nullable;
-
-import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
-import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
-
-/**
- * Cache data row adapter.
- */
-public class CacheDataRowAdapter implements CacheDataRow {
- /** */
- @GridToStringExclude
- protected long link;
-
- /** */
- @GridToStringInclude
- protected KeyCacheObject key;
-
- /** */
- @GridToStringInclude
- protected CacheObject val;
-
- /** */
- protected long expireTime = -1;
-
- /** */
- @GridToStringInclude
- protected GridCacheVersion ver;
-
- /** */
- @GridToStringInclude
- protected int cacheId;
-
- /**
- * @param link Link.
- */
- public CacheDataRowAdapter(long link) {
- // Link can be 0 here.
- this.link = link;
- }
-
- /**
- * @param key Key.
- * @param val Value.
- * @param expireTime Expire time.
- * @param ver Version.
- */
- public CacheDataRowAdapter(KeyCacheObject key, CacheObject val, GridCacheVersion ver, long expireTime) {
- this.key = key;
- this.val = val;
- this.ver = ver;
- this.expireTime = expireTime;
- }
-
- /**
- * Read row from data pages.
- *
- * @param grp Cache group.
- * @param rowData Required row data.
- * @throws IgniteCheckedException If failed.
- */
- public final void initFromLink(CacheGroupContext grp, RowData rowData) throws IgniteCheckedException {
- initFromLink(grp, grp.shared(), grp.memoryPolicy().pageMemory(), rowData);
- }
-
- /**
- * Read row from data pages.
- * Can be called with cctx == null, if cache instance is unknown, but its ID is stored in the data row.
- *
- * @param grp Cache group.
- * @param sharedCtx Shared context.
- * @param pageMem Page memory.
- * @param rowData Row data.
- * @throws IgniteCheckedException If failed.
- */
- public final void initFromLink(
- @Nullable CacheGroupContext grp,
- GridCacheSharedContext<?, ?> sharedCtx,
- PageMemory pageMem,
- RowData rowData)
- throws IgniteCheckedException {
- assert link != 0 : "link";
- assert key == null : "key";
-
- CacheObjectContext coctx = grp != null ? grp.cacheObjectContext() : null;
-
- boolean readCacheId = grp == null || grp.storeCacheIdInDataPage();
-
- long nextLink = link;
- IncompleteObject<?> incomplete = null;
- boolean first = true;
-
- do {
- final long pageId = pageId(nextLink);
-
- // Group is null if try evict page, with persistence evictions should be disabled.
- assert grp != null || !sharedCtx.database().persistenceEnabled();
-
- int grpId = grp != null ? grp.groupId() : 0;
-
- final long page = pageMem.acquirePage(grpId, pageId);
-
- try {
- long pageAddr = pageMem.readLock(grpId, pageId, page); // Non-empty data page must not be recycled.
-
- assert pageAddr != 0L : nextLink;
-
- try {
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
-
- DataPagePayload data = io.readPayload(pageAddr,
- itemId(nextLink),
- pageMem.pageSize());
-
- nextLink = data.nextLink();
-
- if (first) {
- if (nextLink == 0) {
- // Fast path for a single page row.
- readFullRow(sharedCtx, coctx, pageAddr + data.offset(), rowData, readCacheId);
-
- return;
- }
-
- first = false;
- }
-
- ByteBuffer buf = pageMem.pageBuffer(pageAddr);
-
- buf.position(data.offset());
- buf.limit(data.offset() + data.payloadSize());
-
- boolean keyOnly = rowData == RowData.KEY_ONLY;
-
- incomplete = readFragment(sharedCtx, coctx, buf, keyOnly, readCacheId, incomplete);
-
- if (keyOnly && key != null)
- return;
- }
- finally {
- pageMem.readUnlock(grpId, pageId, page);
- }
- }
- finally {
- pageMem.releasePage(grpId, pageId, page);
- }
- }
- while(nextLink != 0);
-
- assert isReady() : "ready";
- }
-
- /**
- * @param sharedCtx Cache shared context.
- * @param coctx Cache object context.
- * @param buf Buffer.
- * @param keyOnly {@code true} If need to read only key object.
- * @param readCacheId {@code true} If need to read cache ID.
- * @param incomplete Incomplete object.
- * @throws IgniteCheckedException If failed.
- * @return Read object.
- */
- private IncompleteObject<?> readFragment(
- GridCacheSharedContext<?, ?> sharedCtx,
- CacheObjectContext coctx,
- ByteBuffer buf,
- boolean keyOnly,
- boolean readCacheId,
- IncompleteObject<?> incomplete
- ) throws IgniteCheckedException {
- if (readCacheId && cacheId == 0) {
- incomplete = readIncompleteCacheId(buf, incomplete);
-
- if (cacheId == 0)
- return incomplete;
-
- incomplete = null;
- }
-
- if (coctx == null) {
- // coctx can be null only when grp is null too, this means that
- // we are in process of eviction and cacheId is mandatory part of data.
- assert cacheId != 0;
-
- coctx = sharedCtx.cacheContext(cacheId).cacheObjectContext();
- }
-
- // Read key.
- if (key == null) {
- incomplete = readIncompleteKey(coctx, buf, (IncompleteCacheObject)incomplete);
-
- if (key == null || keyOnly)
- return incomplete;
-
- incomplete = null;
- }
-
- if (expireTime == -1) {
- incomplete = readIncompleteExpireTime(buf, incomplete);
-
- if (expireTime == -1)
- return incomplete;
-
- incomplete = null;
- }
-
- // Read value.
- if (val == null) {
- incomplete = readIncompleteValue(coctx, buf, (IncompleteCacheObject)incomplete);
-
- if (val == null)
- return incomplete;
-
- incomplete = null;
- }
-
- // Read version.
- if (ver == null)
- incomplete = readIncompleteVersion(buf, incomplete);
-
- return incomplete;
- }
-
- /**
- * @param sharedCtx Cache shared context.
- * @param coctx Cache object context.
- * @param addr Address.
- * @param rowData Required row data.
- * @param readCacheId {@code true} If need to read cache ID.
- * @throws IgniteCheckedException If failed.
- */
- private void readFullRow(
- GridCacheSharedContext<?, ?> sharedCtx,
- CacheObjectContext coctx,
- long addr,
- RowData rowData,
- boolean readCacheId)
- throws IgniteCheckedException {
- int off = 0;
-
- if (readCacheId) {
- cacheId = PageUtils.getInt(addr, off);
-
- off += 4;
- }
-
- if (coctx == null)
- coctx = sharedCtx.cacheContext(cacheId).cacheObjectContext();
-
- int len = PageUtils.getInt(addr, off);
- off += 4;
-
- if (rowData != RowData.NO_KEY) {
- byte type = PageUtils.getByte(addr, off);
- off++;
-
- byte[] bytes = PageUtils.getBytes(addr, off, len);
- off += len;
-
- key = coctx.kernalContext().cacheObjects().toKeyCacheObject(coctx, type, bytes);
-
- if (rowData == RowData.KEY_ONLY)
- return;
- }
- else
- off += len + 1;
-
- len = PageUtils.getInt(addr, off);
- off += 4;
-
- byte type = PageUtils.getByte(addr, off);
- off++;
-
- byte[] bytes = PageUtils.getBytes(addr, off, len);
- off += len;
-
- val = coctx.kernalContext().cacheObjects().toCacheObject(coctx, type, bytes);
-
- ver = CacheVersionIO.read(addr + off, false);
-
- off += CacheVersionIO.size(ver, false);
-
- expireTime = PageUtils.getLong(addr, off);
- }
-
- /**
- * @param buf Buffer.
- * @param incomplete Incomplete.
- */
- private IncompleteObject<?> readIncompleteCacheId(
- ByteBuffer buf,
- IncompleteObject<?> incomplete
- ) {
- if (incomplete == null) {
- int remaining = buf.remaining();
-
- if (remaining == 0)
- return null;
-
- int size = 4;
-
- if (remaining >= size) {
- cacheId = buf.getInt();
-
- assert cacheId != 0;
-
- return null;
- }
-
- incomplete = new IncompleteObject<>(new byte[size]);
- }
-
- incomplete.readData(buf);
-
- if (incomplete.isReady()) {
- final ByteBuffer timeBuf = ByteBuffer.wrap(incomplete.data());
-
- timeBuf.order(buf.order());
-
- cacheId = timeBuf.getInt();
-
- assert cacheId != 0;
- }
-
- return incomplete;
- }
-
- /**
- * @param coctx Cache object context.
- * @param buf Buffer.
- * @param incomplete Incomplete object.
- * @return Incomplete object.
- * @throws IgniteCheckedException If failed.
- */
- private IncompleteCacheObject readIncompleteKey(
- CacheObjectContext coctx,
- ByteBuffer buf,
- IncompleteCacheObject incomplete
- ) throws IgniteCheckedException {
- incomplete = coctx.kernalContext().cacheObjects().toKeyCacheObject(coctx, buf, incomplete);
-
- if (incomplete.isReady()) {
- key = (KeyCacheObject)incomplete.object();
-
- assert key != null;
- }
- else
- assert !buf.hasRemaining();
-
- return incomplete;
- }
-
- /**
- * @param coctx Cache object context.
- * @param buf Buffer.
- * @param incomplete Incomplete object.
- * @return Incomplete object.
- * @throws IgniteCheckedException If failed.
- */
- private IncompleteCacheObject readIncompleteValue(
- CacheObjectContext coctx,
- ByteBuffer buf,
- IncompleteCacheObject incomplete
- ) throws IgniteCheckedException {
- incomplete = coctx.kernalContext().cacheObjects().toCacheObject(coctx, buf, incomplete);
-
- if (incomplete.isReady()) {
- val = incomplete.object();
-
- assert val != null;
- }
- else
- assert !buf.hasRemaining();
-
- return incomplete;
- }
-
- /**
- * @param buf Buffer.
- * @param incomplete Incomplete object.
- * @return Incomplete object.
- */
- private IncompleteObject<?> readIncompleteExpireTime(
- ByteBuffer buf,
- IncompleteObject<?> incomplete
- ) {
- if (incomplete == null) {
- int remaining = buf.remaining();
-
- if (remaining == 0)
- return null;
-
- int size = 8;
-
- if (remaining >= size) {
- expireTime = buf.getLong();
-
- assert expireTime >= 0 : expireTime;
-
- return null;
- }
-
- incomplete = new IncompleteObject<>(new byte[size]);
- }
-
- incomplete.readData(buf);
-
- if (incomplete.isReady()) {
- final ByteBuffer timeBuf = ByteBuffer.wrap(incomplete.data());
-
- timeBuf.order(buf.order());
-
- expireTime = timeBuf.getLong();
-
- assert expireTime >= 0;
- }
-
- return incomplete;
- }
-
- /**
- * @param buf Buffer.
- * @param incomplete Incomplete object.
- * @return Incomplete object.
- * @throws IgniteCheckedException If failed.
- */
- private IncompleteObject<?> readIncompleteVersion(
- ByteBuffer buf,
- IncompleteObject<?> incomplete
- ) throws IgniteCheckedException {
- if (incomplete == null) {
- int remaining = buf.remaining();
-
- if (remaining == 0)
- return null;
-
- int size = CacheVersionIO.readSize(buf, false);
-
- if (remaining >= size) {
- // If the whole version is on a single page, just read it.
- ver = CacheVersionIO.read(buf, false);
-
- assert !buf.hasRemaining(): buf.remaining();
- assert ver != null;
-
- return null;
- }
-
- // We have to read multipart version.
- incomplete = new IncompleteObject<>(new byte[size]);
- }
-
- incomplete.readData(buf);
-
- if (incomplete.isReady()) {
- final ByteBuffer verBuf = ByteBuffer.wrap(incomplete.data());
-
- verBuf.order(buf.order());
-
- ver = CacheVersionIO.read(verBuf, false);
-
- assert ver != null;
- }
-
- assert !buf.hasRemaining();
-
- return incomplete;
- }
-
- /**
- * @return {@code True} if entry is ready.
- */
- public boolean isReady() {
- return ver != null && val != null && key != null;
- }
-
- /** {@inheritDoc} */
- @Override public KeyCacheObject key() {
- assert key != null : "Key is not ready: " + this;
-
- return key;
- }
-
- /**
- * @param key Key.
- */
- @Override public void key(KeyCacheObject key) {
- assert key != null;
-
- this.key = key;
- }
-
- /** {@inheritDoc} */
- @Override public int cacheId() {
- return cacheId;
- }
-
- /** {@inheritDoc} */
- @Override public CacheObject value() {
- assert val != null : "Value is not ready: " + this;
-
- return val;
- }
-
- /** {@inheritDoc} */
- @Override public GridCacheVersion version() {
- assert ver != null : "Version is not ready: " + this;
-
- return ver;
- }
-
- /** {@inheritDoc} */
- @Override public long expireTime() {
- return expireTime;
- }
-
- /** {@inheritDoc} */
- @Override public int partition() {
- return PageIdUtils.partId(link);
- }
-
- /** {@inheritDoc} */
- @Override public long link() {
- return link;
- }
-
- /** {@inheritDoc} */
- @Override public void link(long link) {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public int hash() {
- throw new UnsupportedOperationException();
- }
-
- /**
- *
- */
- public enum RowData {
- /** */
- FULL,
-
- /** */
- KEY_ONLY,
-
- /** */
- NO_KEY
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(CacheDataRowAdapter.class, this, "link", U.hexLong(link));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheSearchRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheSearchRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheSearchRow.java
deleted file mode 100644
index 6e429c4..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CacheSearchRow.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-
-/**
- *
- */
-public interface CacheSearchRow {
- /**
- * @return Cache key.
- */
- public KeyCacheObject key();
-
- /**
- * @return Link for this row.
- */
- public long link();
-
- /**
- * @return Key hash code.
- */
- public int hash();
-
- /**
- * @return Cache ID or {@code 0} if cache ID is not defined.
- */
- public int cacheId();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CheckpointLockStateChecker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CheckpointLockStateChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CheckpointLockStateChecker.java
deleted file mode 100644
index df90f33..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/CheckpointLockStateChecker.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-/**
- * Interface for perform checking that checkpoint lock is held by current track
- */
-public interface CheckpointLockStateChecker {
- /**
- * @return true if checkpoint lock is held by current thread
- */
- public boolean checkpointLockIsHeldByThread();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/DataStructure.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/DataStructure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/DataStructure.java
deleted file mode 100644
index 0e35bf4..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/DataStructure.java
+++ /dev/null
@@ -1,384 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageLockListener;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.MAX_PARTITION_ID;
-
-/**
- * Base class for all the data structures based on {@link PageMemory}.
- */
-public abstract class DataStructure implements PageLockListener {
- /** For tests. */
- public static Random rnd;
-
- /** */
- protected final int cacheId;
-
- /** */
- protected final PageMemory pageMem;
-
- /** */
- protected final IgniteWriteAheadLogManager wal;
-
- /** */
- protected ReuseList reuseList;
-
- /**
- * @param cacheId Cache ID.
- * @param pageMem Page memory.
- * @param wal Write ahead log manager.
- */
- public DataStructure(
- int cacheId,
- PageMemory pageMem,
- IgniteWriteAheadLogManager wal
- ) {
- assert pageMem != null;
-
- this.cacheId = cacheId;
- this.pageMem = pageMem;
- this.wal = wal;
- }
-
- /**
- * @return Cache ID.
- */
- public final int getCacheId() {
- return cacheId;
- }
-
- /**
- * @param max Max.
- * @return Random value from {@code 0} (inclusive) to the given max value (exclusive).
- */
- public static int randomInt(int max) {
- Random rnd0 = rnd != null ? rnd : ThreadLocalRandom.current();
-
- return rnd0.nextInt(max);
- }
-
- /**
- * @param bag Reuse bag.
- * @return Allocated page.
- * @throws IgniteCheckedException If failed.
- */
- protected final long allocatePage(ReuseBag bag) throws IgniteCheckedException {
- long pageId = bag != null ? bag.pollFreePage() : 0;
-
- if (pageId == 0 && reuseList != null)
- pageId = reuseList.takeRecycledPage();
-
- if (pageId == 0)
- pageId = allocatePageNoReuse();
-
- assert pageId != 0;
-
- return pageId;
- }
-
- /**
- * @return Page ID of newly allocated page.
- * @throws IgniteCheckedException If failed.
- */
- protected long allocatePageNoReuse() throws IgniteCheckedException {
- return pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, FLAG_IDX);
- }
-
- /**
- * @param pageId Page ID.
- * @return Page absolute pointer.
- * @throws IgniteCheckedException If failed.
- */
- protected final long acquirePage(long pageId) throws IgniteCheckedException {
- assert PageIdUtils.flag(pageId) == FLAG_IDX && PageIdUtils.partId(pageId) == INDEX_PARTITION ||
- PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID : U.hexLong(pageId);
-
- return pageMem.acquirePage(cacheId, pageId);
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- */
- protected final void releasePage(long pageId, long page) {
- pageMem.releasePage(cacheId, pageId, page);
- }
-
- /**
- * @param pageId Page ID
- * @param page Page pointer.
- * @return Page address or {@code 0} if failed to lock due to recycling.
- */
- protected final long tryWriteLock(long pageId, long page) {
- return PageHandler.writeLock(pageMem, cacheId, pageId, page, this, true);
- }
-
- /**
- * @param pageId Page ID
- * @param page Page pointer.
- * @return Page address.
- */
- protected final long writeLock(long pageId, long page) {
- return PageHandler.writeLock(pageMem, cacheId, pageId, page, this, false);
- }
-
- /**
- * <p>
- * Note: Default WAL record policy will be used.
- * </p>
- * @param pageId Page ID
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param dirty Dirty flag.
- */
- protected final void writeUnlock(long pageId, long page, long pageAddr, boolean dirty) {
- writeUnlock(pageId, page, pageAddr, null, dirty);
- }
-
- /**
- * @param pageId Page ID
- * @param page Page pointer.
- * @return Page address.
- */
- protected final long readLock(long pageId, long page) {
- return PageHandler.readLock(pageMem, cacheId, pageId, page, this);
- }
-
- /**
- * @param pageId Page ID
- * @param page Page pointer.
- * @param pageAddr Page address.
- */
- protected final void readUnlock(long pageId, long page, long pageAddr) {
- PageHandler.readUnlock(pageMem, cacheId, pageId, page, pageAddr, this);
- }
-
- /**
- * @param pageId Page ID
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param walPlc Full page WAL record policy.
- * @param dirty Dirty flag.
- */
- protected final void writeUnlock(long pageId, long page, long pageAddr, Boolean walPlc, boolean dirty) {
- PageHandler.writeUnlock(pageMem, cacheId, pageId, page, pageAddr, this, walPlc, dirty);
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param walPlc Full page WAL record policy.
- * @return {@code true} If we need to make a delta WAL record for the change in this page.
- */
- protected final boolean needWalDeltaRecord(long pageId, long page, Boolean walPlc) {
- return PageHandler.isWalDeltaRecordNeeded(pageMem, cacheId, pageId, page, wal, walPlc);
- }
-
- /**
- * @param pageId Page ID.
- * @param h Handler.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- protected final <R> R write(
- long pageId,
- PageHandler<?, R> h,
- int intArg,
- R lockFailed) throws IgniteCheckedException {
- return PageHandler.writePage(pageMem, cacheId, pageId, this, h, null, null, null, null, intArg, lockFailed);
- }
-
- /**
- * @param pageId Page ID.
- * @param h Handler.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- protected final <X, R> R write(
- long pageId,
- PageHandler<X, R> h,
- X arg,
- int intArg,
- R lockFailed) throws IgniteCheckedException {
- return PageHandler.writePage(pageMem, cacheId, pageId, this, h, null, null, null, arg, intArg, lockFailed);
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param h Handler.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- protected final <X, R> R write(
- long pageId,
- long page,
- PageHandler<X, R> h,
- X arg,
- int intArg,
- R lockFailed) throws IgniteCheckedException {
- return PageHandler.writePage(pageMem, cacheId, pageId, page, this, h, null, null, null, arg, intArg, lockFailed);
- }
-
- /**
- * @param pageId Page ID.
- * @param h Handler.
- * @param init IO for new page initialization or {@code null} if it is an existing page.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- protected final <X, R> R write(
- long pageId,
- PageHandler<X, R> h,
- PageIO init,
- X arg,
- int intArg,
- R lockFailed) throws IgniteCheckedException {
- return PageHandler.writePage(pageMem, cacheId, pageId, this, h, init, wal, null, arg, intArg, lockFailed);
- }
-
- /**
- * @param pageId Page ID.
- * @param h Handler.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- protected final <X, R> R read(
- long pageId,
- PageHandler<X, R> h,
- X arg,
- int intArg,
- R lockFailed) throws IgniteCheckedException {
- return PageHandler.readPage(pageMem, cacheId, pageId, this, h, arg, intArg, lockFailed);
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param h Handler.
- * @param arg Argument.
- * @param intArg Argument of type {@code int}.
- * @param lockFailed Result in case of lock failure due to page recycling.
- * @return Handler result.
- * @throws IgniteCheckedException If failed.
- */
- protected final <X, R> R read(
- long pageId,
- long page,
- PageHandler<X, R> h,
- X arg,
- int intArg,
- R lockFailed) throws IgniteCheckedException {
- return PageHandler.readPage(pageMem, cacheId, pageId, page, this, h, arg, intArg, lockFailed);
- }
-
- /**
- * @param pageId Page ID.
- * @param init IO for new page initialization.
- * @throws IgniteCheckedException if failed.
- */
- protected final void init(long pageId, PageIO init) throws IgniteCheckedException {
- PageHandler.initPage(pageMem, cacheId, pageId, init, wal, this);
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param walPlc Full page WAL record policy.
- * @return Rotated page ID.
- * @throws IgniteCheckedException If failed.
- */
- protected final long recyclePage(
- long pageId,
- long page,
- long pageAddr,
- Boolean walPlc) throws IgniteCheckedException {
- long rotated = PageIdUtils.rotatePageId(pageId);
-
- PageIO.setPageId(pageAddr, rotated);
-
- if (needWalDeltaRecord(pageId, page, walPlc))
- wal.log(new RecycleRecord(cacheId, pageId, rotated));
-
- return rotated;
- }
-
- /**
- * @return Page size.
- */
- protected final int pageSize() {
- return pageMem.pageSize();
- }
-
- @Override public void onBeforeWriteLock(int cacheId, long pageId, long page) {
- // No-op.
- }
-
- @Override public void onWriteLock(int cacheId, long pageId, long page, long pageAddr) {
- // No-op.
- }
-
- @Override public void onWriteUnlock(int cacheId, long pageId, long page, long pageAddr) {
- // No-op.
- }
-
- @Override public void onBeforeReadLock(int cacheId, long pageId, long page) {
- // No-op.
- }
-
- @Override public void onReadLock(int cacheId, long pageId, long page, long pageAddr) {
- // No-op.
- }
-
- @Override public void onReadUnlock(int cacheId, long pageId, long page, long pageAddr) {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
deleted file mode 100644
index a4ca5d2..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheDatabaseSharedManager.java
+++ /dev/null
@@ -1,968 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.io.File;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import javax.management.JMException;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.MemoryMetrics;
-import org.apache.ignite.PersistenceMetrics;
-import org.apache.ignite.configuration.DataPageEvictionMode;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheMapEntry;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
-import org.apache.ignite.internal.processors.cache.database.evict.FairFifoPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.evict.NoOpPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.evict.PageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.evict.Random2LruPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.evict.RandomLruPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeListImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
-import org.apache.ignite.internal.processors.cluster.IgniteChangeGlobalStateSupport;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.LT;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.mxbean.MemoryMetricsMXBean;
-import org.jetbrains.annotations.Nullable;
-
-import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE;
-import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
-
-/**
- *
- */
-public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdapter
- implements IgniteChangeGlobalStateSupport, CheckpointLockStateChecker {
- /** MemoryPolicyConfiguration name reserved for internal caches. */
- static final String SYSTEM_MEMORY_POLICY_NAME = "sysMemPlc";
-
- /** Minimum size of memory chunk */
- private static final long MIN_PAGE_MEMORY_SIZE = 10 * 1024 * 1024;
-
- /** Maximum initial size on 32-bit JVM */
- private static final long MAX_PAGE_MEMORY_INIT_SIZE_32_BIT = 2L * 1024 * 1024 * 1024;
-
- /** */
- protected Map<String, MemoryPolicy> memPlcMap;
-
- /** */
- protected Map<String, MemoryMetrics> memMetricsMap;
-
- /** */
- protected MemoryPolicy dfltMemPlc;
-
- /** */
- private Map<String, FreeListImpl> freeListMap;
-
- /** */
- private FreeListImpl dfltFreeList;
-
- /** */
- private int pageSize;
-
- /** {@inheritDoc} */
- @Override protected void start0() throws IgniteCheckedException {
- if (cctx.kernalContext().clientNode() && cctx.kernalContext().config().getMemoryConfiguration() == null)
- return;
-
- init();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void init() throws IgniteCheckedException {
- MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration();
-
- assert memCfg != null;
-
- validateConfiguration(memCfg);
-
- pageSize = memCfg.getPageSize();
-
- initPageMemoryPolicies(memCfg);
-
- //registerMetricsMBeans();
-
- startMemoryPolicies();
-
- initPageMemoryDataStructures(memCfg);
- }
-
- /**
- * Registers MBeans for all MemoryMetrics configured in this instance.
- */
- private void registerMetricsMBeans() {
- IgniteConfiguration cfg = cctx.gridConfig();
-
- for (MemoryMetrics memMetrics : memMetricsMap.values()) {
- MemoryPolicyConfiguration memPlcCfg = memPlcMap.get(memMetrics.getName()).config();
-
- registerMetricsMBean((MemoryMetricsImpl)memMetrics, memPlcCfg, cfg);
- }
- }
-
- /**
- * @param memMetrics Memory metrics.
- * @param memPlcCfg Memory policy configuration.
- * @param cfg Ignite configuration.
- */
- private void registerMetricsMBean(
- MemoryMetricsImpl memMetrics,
- MemoryPolicyConfiguration memPlcCfg,
- IgniteConfiguration cfg
- ) {
- try {
- U.registerMBean(
- cfg.getMBeanServer(),
- cfg.getIgniteInstanceName(),
- "MemoryMetrics",
- memPlcCfg.getName(),
- new MemoryMetricsMXBeanImpl(memMetrics, memPlcCfg),
- MemoryMetricsMXBean.class);
- }
- catch (JMException e) {
- U.error(log, "Failed to register MBean for MemoryMetrics with name: '" + memMetrics.getName() + "'", e);
- }
- }
-
- /**
- * @param dbCfg Database config.
- */
- protected void initPageMemoryDataStructures(MemoryConfiguration dbCfg) throws IgniteCheckedException {
- freeListMap = U.newHashMap(memPlcMap.size());
-
- String dfltMemPlcName = dbCfg.getDefaultMemoryPolicyName();
-
- for (MemoryPolicy memPlc : memPlcMap.values()) {
- MemoryPolicyConfiguration memPlcCfg = memPlc.config();
-
- MemoryMetricsImpl memMetrics = (MemoryMetricsImpl) memMetricsMap.get(memPlcCfg.getName());
-
- FreeListImpl freeList = new FreeListImpl(0,
- cctx.igniteInstanceName(),
- memMetrics,
- memPlc,
- null,
- cctx.wal(),
- 0L,
- true);
-
- memMetrics.freeList(freeList);
-
- freeListMap.put(memPlcCfg.getName(), freeList);
- }
-
- dfltFreeList = freeListMap.get(dfltMemPlcName);
- }
-
- /**
- * @return Size of page used for PageMemory regions.
- */
- public int pageSize() {
- return pageSize;
- }
-
- /**
- *
- */
- private void startMemoryPolicies() {
- for (MemoryPolicy memPlc : memPlcMap.values()) {
- memPlc.pageMemory().start();
-
- memPlc.evictionTracker().start();
- }
- }
-
- /**
- * @param memCfg Database config.
- */
- protected void initPageMemoryPolicies(MemoryConfiguration memCfg) {
- MemoryPolicyConfiguration[] memPlcsCfgs = memCfg.getMemoryPolicies();
-
- if (memPlcsCfgs == null) {
- //reserve place for default and system memory policies
- memPlcMap = U.newHashMap(2);
- memMetricsMap = U.newHashMap(2);
-
- addMemoryPolicy(
- memCfg,
- memCfg.createDefaultPolicyConfig(),
- DFLT_MEM_PLC_DEFAULT_NAME
- );
-
- U.warn(log, "No user-defined default MemoryPolicy found; system default of 1GB size will be used.");
- }
- else {
- String dfltMemPlcName = memCfg.getDefaultMemoryPolicyName();
-
- if (DFLT_MEM_PLC_DEFAULT_NAME.equals(dfltMemPlcName) && !hasCustomDefaultMemoryPolicy(memPlcsCfgs)) {
- //reserve additional place for default and system memory policies
- memPlcMap = U.newHashMap(memPlcsCfgs.length + 2);
- memMetricsMap = U.newHashMap(memPlcsCfgs.length + 2);
-
- addMemoryPolicy(
- memCfg,
- memCfg.createDefaultPolicyConfig(),
- DFLT_MEM_PLC_DEFAULT_NAME
- );
-
- U.warn(log, "No user-defined default MemoryPolicy found; system default of 1GB size will be used.");
- }
- else {
- //reserve additional space for system memory policy only
- memPlcMap = U.newHashMap(memPlcsCfgs.length + 1);
- memMetricsMap = U.newHashMap(memPlcsCfgs.length + 1);
- }
-
- for (MemoryPolicyConfiguration memPlcCfg : memPlcsCfgs)
- addMemoryPolicy(memCfg, memPlcCfg, memPlcCfg.getName());
- }
-
- addMemoryPolicy(
- memCfg,
- createSystemMemoryPolicy(
- memCfg.getSystemCacheInitialSize(),
- memCfg.getSystemCacheMaxSize()
- ),
- SYSTEM_MEMORY_POLICY_NAME
- );
- }
-
- /**
- * @param memCfg Database config.
- * @param memPlcCfg Memory policy config.
- * @param memPlcName Memory policy name.
- */
- private void addMemoryPolicy(
- MemoryConfiguration memCfg,
- MemoryPolicyConfiguration memPlcCfg,
- String memPlcName
- ) {
- String dfltMemPlcName = memCfg.getDefaultMemoryPolicyName();
-
- if (dfltMemPlcName == null)
- dfltMemPlcName = DFLT_MEM_PLC_DEFAULT_NAME;
-
- MemoryMetricsImpl memMetrics = new MemoryMetricsImpl(memPlcCfg);
-
- MemoryPolicy memPlc = initMemory(memCfg, memPlcCfg, memMetrics);
-
- memPlcMap.put(memPlcName, memPlc);
-
- memMetricsMap.put(memPlcName, memMetrics);
-
- if (memPlcName.equals(dfltMemPlcName))
- dfltMemPlc = memPlc;
- else if (memPlcName.equals(DFLT_MEM_PLC_DEFAULT_NAME))
- U.warn(log, "Memory Policy with name 'default' isn't used as a default. " +
- "Please check Memory Policies configuration.");
- }
-
- /**
- * @param memPlcsCfgs User-defined memory policy configurations.
- */
- private boolean hasCustomDefaultMemoryPolicy(MemoryPolicyConfiguration[] memPlcsCfgs) {
- for (MemoryPolicyConfiguration memPlcsCfg : memPlcsCfgs) {
- if (DFLT_MEM_PLC_DEFAULT_NAME.equals(memPlcsCfg.getName()))
- return true;
- }
-
- return false;
- }
-
- /**
- * @param dbCfg Database configuration.
- * @param memPlcCfg MemoryPolicy configuration.
- * @param memMetrics MemoryMetrics instance.
- */
- private MemoryPolicy createDefaultMemoryPolicy(MemoryConfiguration dbCfg, MemoryPolicyConfiguration memPlcCfg, MemoryMetricsImpl memMetrics) {
- return initMemory(dbCfg, memPlcCfg, memMetrics);
- }
-
- /**
- * @param sysCacheInitSize Initial size of PageMemory to be created for system cache.
- * @param sysCacheMaxSize Maximum size of PageMemory to be created for system cache.
- *
- * @return {@link MemoryPolicyConfiguration configuration} of MemoryPolicy for system cache.
- */
- private MemoryPolicyConfiguration createSystemMemoryPolicy(long sysCacheInitSize, long sysCacheMaxSize) {
- MemoryPolicyConfiguration res = new MemoryPolicyConfiguration();
-
- res.setName(SYSTEM_MEMORY_POLICY_NAME);
- res.setInitialSize(sysCacheInitSize);
- res.setMaxSize(sysCacheMaxSize);
-
- return res;
- }
-
- /**
- * @param memCfg configuration to validate.
- */
- private void validateConfiguration(MemoryConfiguration memCfg) throws IgniteCheckedException {
- MemoryPolicyConfiguration[] plcCfgs = memCfg.getMemoryPolicies();
-
- Set<String> plcNames = (plcCfgs != null) ? U.<String>newHashSet(plcCfgs.length) : new HashSet<String>(0);
-
- checkSystemMemoryPolicySizeConfiguration(
- memCfg.getSystemCacheInitialSize(),
- memCfg.getSystemCacheMaxSize()
- );
-
- if (plcCfgs != null) {
- for (MemoryPolicyConfiguration plcCfg : plcCfgs) {
- assert plcCfg != null;
-
- checkPolicyName(plcCfg.getName(), plcNames);
-
- checkPolicySize(plcCfg);
-
- checkMetricsProperties(plcCfg);
-
- checkPolicyEvictionProperties(plcCfg, memCfg);
- }
- }
-
- checkDefaultPolicyConfiguration(
- memCfg.getDefaultMemoryPolicyName(),
- memCfg.getDefaultMemoryPolicySize(),
- plcNames
- );
- }
-
- /**
- * @param plcCfg Memory policy config.
- *
- * @throws IgniteCheckedException if validation of memory metrics properties fails.
- */
- private static void checkMetricsProperties(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException {
- if (plcCfg.getRateTimeInterval() <= 0)
- throw new IgniteCheckedException("Rate time interval must be greater than zero " +
- "(use MemoryPolicyConfiguration.rateTimeInterval property to adjust the interval) " +
- "[name=" + plcCfg.getName() +
- ", rateTimeInterval=" + plcCfg.getRateTimeInterval() + "]"
- );
- if (plcCfg.getSubIntervals() <= 0)
- throw new IgniteCheckedException("Sub intervals must be greater than zero " +
- "(use MemoryPolicyConfiguration.subIntervals property to adjust the sub intervals) " +
- "[name=" + plcCfg.getName() +
- ", subIntervals=" + plcCfg.getSubIntervals() + "]"
- );
-
- if (plcCfg.getRateTimeInterval() < 1_000)
- throw new IgniteCheckedException("Rate time interval must be longer that 1 second (1_000 milliseconds) " +
- "(use MemoryPolicyConfiguration.rateTimeInterval property to adjust the interval) " +
- "[name=" + plcCfg.getName() +
- ", rateTimeInterval=" + plcCfg.getRateTimeInterval() + "]");
- }
-
- /**
- * @param sysCacheInitSize System cache initial size.
- * @param sysCacheMaxSize System cache max size.
- *
- * @throws IgniteCheckedException In case of validation violation.
- */
- private static void checkSystemMemoryPolicySizeConfiguration(
- long sysCacheInitSize,
- long sysCacheMaxSize
- ) throws IgniteCheckedException {
- if (sysCacheInitSize < MIN_PAGE_MEMORY_SIZE)
- throw new IgniteCheckedException("Initial size for system cache must have size more than 10MB (use " +
- "MemoryConfiguration.systemCacheInitialSize property to set correct size in bytes) " +
- "[size=" + U.readableSize(sysCacheInitSize, true) + ']'
- );
-
- if (U.jvm32Bit() && sysCacheInitSize > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT)
- throw new IgniteCheckedException("Initial size for system cache exceeds 2GB on 32-bit JVM (use " +
- "MemoryPolicyConfiguration.systemCacheInitialSize property to set correct size in bytes " +
- "or use 64-bit JVM) [size=" + U.readableSize(sysCacheInitSize, true) + ']'
- );
-
- if (sysCacheMaxSize < sysCacheInitSize)
- throw new IgniteCheckedException("MaxSize of system cache must not be smaller than " +
- "initialSize [initSize=" + U.readableSize(sysCacheInitSize, true) +
- ", maxSize=" + U.readableSize(sysCacheMaxSize, true) + "]. " +
- "Use MemoryConfiguration.systemCacheInitialSize/MemoryConfiguration.systemCacheMaxSize " +
- "properties to set correct sizes in bytes."
- );
- }
-
- /**
- * @param dfltPlcName Default MemoryPolicy name.
- * @param dfltPlcSize Default size of MemoryPolicy overridden by user (equals to -1 if wasn't specified by user).
- * @param plcNames All MemoryPolicy names.
- * @throws IgniteCheckedException In case of validation violation.
- */
- private static void checkDefaultPolicyConfiguration(
- String dfltPlcName,
- long dfltPlcSize,
- Collection<String> plcNames
- ) throws IgniteCheckedException {
- if (dfltPlcSize != MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE) {
- if (!F.eq(dfltPlcName, MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME))
- throw new IgniteCheckedException("User-defined MemoryPolicy configuration " +
- "and defaultMemoryPolicySize properties are set at the same time. " +
- "Delete either MemoryConfiguration.defaultMemoryPolicySize property " +
- "or user-defined default MemoryPolicy configuration");
-
- if (dfltPlcSize < MIN_PAGE_MEMORY_SIZE)
- throw new IgniteCheckedException("User-defined default MemoryPolicy size is less than 1MB. " +
- "Use MemoryConfiguration.defaultMemoryPolicySize property to set correct size.");
-
- if (U.jvm32Bit() && dfltPlcSize > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT)
- throw new IgniteCheckedException("User-defined default MemoryPolicy size exceeds 2GB on 32-bit JVM " +
- "(use MemoryConfiguration.defaultMemoryPolicySize property to set correct size in bytes " +
- "or use 64-bit JVM) [size=" + U.readableSize(dfltPlcSize, true) + ']'
- );
- }
-
- if (!DFLT_MEM_PLC_DEFAULT_NAME.equals(dfltPlcName)) {
- if (dfltPlcName.isEmpty())
- throw new IgniteCheckedException("User-defined default MemoryPolicy name must be non-empty");
-
- if (!plcNames.contains(dfltPlcName))
- throw new IgniteCheckedException("User-defined default MemoryPolicy name " +
- "must be presented among configured MemoryPolices: " + dfltPlcName);
- }
- }
-
- /**
- * @param plcCfg MemoryPolicyConfiguration to validate.
- * @throws IgniteCheckedException If config is invalid.
- */
- private void checkPolicySize(MemoryPolicyConfiguration plcCfg) throws IgniteCheckedException {
- if (plcCfg.getInitialSize() < MIN_PAGE_MEMORY_SIZE)
- throw new IgniteCheckedException("MemoryPolicy must have size more than 10MB (use " +
- "MemoryPolicyConfiguration.initialSize property to set correct size in bytes) " +
- "[name=" + plcCfg.getName() + ", size=" + U.readableSize(plcCfg.getInitialSize(), true) + "]"
- );
-
- if (plcCfg.getMaxSize() < plcCfg.getInitialSize()) {
- // We will know for sure if initialSize has been changed if we compare Longs by "==".
- if (plcCfg.getInitialSize() == DFLT_MEMORY_POLICY_INITIAL_SIZE) {
- plcCfg.setInitialSize(plcCfg.getMaxSize());
-
- LT.warn(log, "MemoryPolicy maxSize=" + U.readableSize(plcCfg.getMaxSize(), true) +
- " is smaller than defaultInitialSize=" +
- U.readableSize(MemoryConfiguration.DFLT_MEMORY_POLICY_INITIAL_SIZE, true) +
- ", setting initialSize to " + U.readableSize(plcCfg.getMaxSize(), true));
- }
- else {
- throw new IgniteCheckedException("MemoryPolicy maxSize must not be smaller than " +
- "initialSize [name=" + plcCfg.getName() +
- ", initSize=" + U.readableSize(plcCfg.getInitialSize(), true) +
- ", maxSize=" + U.readableSize(plcCfg.getMaxSize(), true) + ']');
- }
- }
-
- if (U.jvm32Bit() && plcCfg.getInitialSize() > MAX_PAGE_MEMORY_INIT_SIZE_32_BIT)
- throw new IgniteCheckedException("MemoryPolicy initialSize exceeds 2GB on 32-bit JVM (use " +
- "MemoryPolicyConfiguration.initialSize property to set correct size in bytes or use 64-bit JVM) " +
- "[name=" + plcCfg.getName() +
- ", size=" + U.readableSize(plcCfg.getInitialSize(), true) + "]");
- }
-
- /**
- * @param plcCfg MemoryPolicyConfiguration to validate.
- * @param dbCfg Memory configuration.
- * @throws IgniteCheckedException If config is invalid.
- */
- protected void checkPolicyEvictionProperties(MemoryPolicyConfiguration plcCfg, MemoryConfiguration dbCfg)
- throws IgniteCheckedException {
- if (plcCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED)
- return;
-
- if (plcCfg.getEvictionThreshold() < 0.5 || plcCfg.getEvictionThreshold() > 0.999) {
- throw new IgniteCheckedException("Page eviction threshold must be between 0.5 and 0.999: " +
- plcCfg.getName());
- }
-
- if (plcCfg.getEmptyPagesPoolSize() <= 10)
- throw new IgniteCheckedException("Evicted pages pool size should be greater than 10: " + plcCfg.getName());
-
- long maxPoolSize = plcCfg.getMaxSize() / dbCfg.getPageSize() / 10;
-
- if (plcCfg.getEmptyPagesPoolSize() >= maxPoolSize) {
- throw new IgniteCheckedException("Evicted pages pool size should be lesser than " + maxPoolSize +
- ": " + plcCfg.getName());
- }
- }
-
- /**
- * @param plcName MemoryPolicy name to validate.
- * @param observedNames Names of MemoryPolicies observed before.
- * @throws IgniteCheckedException If config is invalid.
- */
- private static void checkPolicyName(String plcName, Collection<String> observedNames)
- throws IgniteCheckedException {
- if (plcName == null || plcName.isEmpty())
- throw new IgniteCheckedException("User-defined MemoryPolicyConfiguration must have non-null and " +
- "non-empty name.");
-
- if (observedNames.contains(plcName))
- throw new IgniteCheckedException("Two MemoryPolicies have the same name: " + plcName);
-
- if (SYSTEM_MEMORY_POLICY_NAME.equals(plcName))
- throw new IgniteCheckedException("'sysMemPlc' policy name is reserved for internal use.");
-
- observedNames.add(plcName);
- }
-
- /**
- * @param log Logger.
- */
- public void dumpStatistics(IgniteLogger log) {
- if (freeListMap != null) {
- for (FreeListImpl freeList : freeListMap.values())
- freeList.dumpStatistics(log);
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void initDataBase() throws IgniteCheckedException {
- // No-op.
- }
-
- /**
- * @return collection of all configured {@link MemoryPolicy policies}.
- */
- public Collection<MemoryPolicy> memoryPolicies() {
- return memPlcMap != null ? memPlcMap.values() : null;
- }
-
- /**
- * @return MemoryMetrics for all MemoryPolicies configured in Ignite instance.
- */
- public Collection<MemoryMetrics> memoryMetrics() {
- if (!F.isEmpty(memMetricsMap)) {
- // Intentionally return a collection copy to make it explicitly serializable.
- Collection<MemoryMetrics> res = new ArrayList<>(memMetricsMap.size());
-
- for (MemoryMetrics metrics : memMetricsMap.values())
- res.add(new MemoryMetricsSnapshot(metrics));
-
- return res;
- }
- else
- return Collections.emptyList();
- }
-
- /**
- * @return PersistenceMetrics if persistence is enabled or {@code null} otherwise.
- */
- public PersistenceMetrics persistentStoreMetrics() {
- return null;
- }
-
- /**
- * @param memPlcName Name of {@link MemoryPolicy} to obtain {@link MemoryMetrics} for.
- * @return {@link MemoryMetrics} snapshot for specified {@link MemoryPolicy} or {@code null} if
- * no {@link MemoryPolicy} is configured for specified name.
- */
- @Nullable public MemoryMetrics memoryMetrics(String memPlcName) {
- if (!F.isEmpty(memMetricsMap)) {
- MemoryMetrics memMetrics = memMetricsMap.get(memPlcName);
-
- if (memMetrics == null)
- return null;
- else
- return new MemoryMetricsSnapshot(memMetrics);
- }
- else
- return null;
- }
-
- /**
- * @param memPlcName Memory policy name.
- * @return {@link MemoryPolicy} instance associated with a given {@link MemoryPolicyConfiguration}.
- * @throws IgniteCheckedException in case of request for unknown MemoryPolicy.
- */
- public MemoryPolicy memoryPolicy(String memPlcName) throws IgniteCheckedException {
- if (memPlcName == null)
- return dfltMemPlc;
-
- if (memPlcMap == null)
- return null;
-
- MemoryPolicy plc;
-
- if ((plc = memPlcMap.get(memPlcName)) == null)
- throw new IgniteCheckedException("Requested MemoryPolicy is not configured: " + memPlcName);
-
- return plc;
- }
-
- /**
- * @param memPlcName MemoryPolicyConfiguration name.
- * @return {@link FreeList} instance associated with a given {@link MemoryPolicyConfiguration}.
- */
- public FreeList freeList(String memPlcName) {
- if (memPlcName == null)
- return dfltFreeList;
-
- return freeListMap != null ? freeListMap.get(memPlcName) : null;
- }
-
- /**
- * @param memPlcName MemoryPolicyConfiguration name.
- * @return {@link ReuseList} instance associated with a given {@link MemoryPolicyConfiguration}.
- */
- public ReuseList reuseList(String memPlcName) {
- if (memPlcName == null)
- return dfltFreeList;
-
- return freeListMap != null ? freeListMap.get(memPlcName) : null;
- }
-
- /** {@inheritDoc} */
- @Override protected void stop0(boolean cancel) {
- if (memPlcMap != null) {
- for (MemoryPolicy memPlc : memPlcMap.values()) {
- memPlc.pageMemory().stop();
-
- memPlc.evictionTracker().stop();
-
- IgniteConfiguration cfg = cctx.gridConfig();
-
- /* try {
- cfg.getMBeanServer().unregisterMBean(
- U.makeMBeanName(
- cfg.getIgniteInstanceName(),
- "MemoryMetrics",
- memPlc.memoryMetrics().getName()));
- }
- catch (JMException e) {
- U.error(log, "Failed to unregister MBean for memory metrics: " +
- memPlc.memoryMetrics().getName(), e);
- }*/
- }
- }
- }
-
- /**
- *
- */
- public boolean persistenceEnabled() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public boolean checkpointLockIsHeldByThread() {
- return false;
- }
-
- /**
- *
- */
- public void lock() throws IgniteCheckedException {
-
- }
-
- /**
- *
- */
- public void unLock() {
-
- }
-
- /**
- * No-op for non-persistent storage.
- */
- public void checkpointReadLock() {
- // No-op.
- }
-
- /**
- * No-op for non-persistent storage.
- */
- public void checkpointReadUnlock() {
- // No-op.
- }
-
- /**
- *
- */
- @Nullable public IgniteInternalFuture wakeupForCheckpoint(String reason) {
- return null;
- }
-
- /**
- * Waits until current state is checkpointed.
- *
- * @throws IgniteCheckedException If failed.
- */
- public void waitForCheckpoint(String reason) throws IgniteCheckedException {
- // No-op
- }
-
- /**
- * @param discoEvt Before exchange for the given discovery event.
- */
- public void beforeExchange(GridDhtPartitionsExchangeFuture discoEvt) throws IgniteCheckedException {
- // No-op.
- }
-
- /**
- * Needed action before any cache will stop
- */
- public void prepareCachesStop() {
- // No-op.
- }
-
- /**
- * @param stoppedGrps A collection of tuples (cache group, destroy flag).
- */
- public void onCacheGroupsStopped(Collection<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGrps) {
- // No-op.
- }
-
- /**
- * @param cctx Stopped cache context.
- */
- public void onCacheStop(GridCacheContext cctx) {
- // No-op
- }
-
- /**
- * @return Future that will be completed when indexes for given cache are restored.
- */
- @Nullable public IgniteInternalFuture indexRebuildFuture(int cacheId) {
- return null;
- }
-
- /**
- * Reserve update history for exchange.
- *
- * @return Reserved update counters per cache and partition.
- */
- public Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
- return Collections.emptyMap();
- }
-
- /**
- * Release reserved update history.
- */
- public void releaseHistoryForExchange() {
- // No-op
- }
-
- /**
- * Reserve update history for preloading.
- * @param grpId Cache group ID.
- * @param partId Partition Id.
- * @param cntr Update counter.
- * @return True if successfully reserved.
- */
- public boolean reserveHistoryForPreloading(int grpId, int partId, long cntr) {
- return false;
- }
-
- /**
- * Release reserved update history.
- */
- public void releaseHistoryForPreloading() {
- // No-op
- }
-
- /**
- * See {@link GridCacheMapEntry#ensureFreeSpace()}
- *
- * @param memPlc Memory policy.
- */
- public void ensureFreeSpace(MemoryPolicy memPlc) throws IgniteCheckedException {
- if (memPlc == null)
- return;
-
- MemoryPolicyConfiguration plcCfg = memPlc.config();
-
- if (plcCfg.getPageEvictionMode() == DataPageEvictionMode.DISABLED)
- return;
-
- long memorySize = plcCfg.getMaxSize();
-
- PageMemory pageMem = memPlc.pageMemory();
-
- int sysPageSize = pageMem.systemPageSize();
-
- FreeListImpl freeListImpl = freeListMap.get(plcCfg.getName());
-
- for (;;) {
- long allocatedPagesCnt = pageMem.loadedPages();
-
- int emptyDataPagesCnt = freeListImpl.emptyDataPages();
-
- boolean shouldEvict = allocatedPagesCnt > (memorySize / sysPageSize * plcCfg.getEvictionThreshold()) &&
- emptyDataPagesCnt < plcCfg.getEmptyPagesPoolSize();
-
- if (shouldEvict)
- memPlc.evictionTracker().evictDataPage();
- else
- break;
- }
- }
-
- /**
- * @param memCfg memory configuration with common parameters.
- * @param plcCfg memory policy with PageMemory specific parameters.
- * @param memMetrics {@link MemoryMetrics} object to collect memory usage metrics.
- * @return Memory policy instance.
- */
- private MemoryPolicy initMemory(
- MemoryConfiguration memCfg,
- MemoryPolicyConfiguration plcCfg,
- MemoryMetricsImpl memMetrics
- ) {
- File allocPath = buildAllocPath(plcCfg);
-
- DirectMemoryProvider memProvider = allocPath == null ?
- new UnsafeMemoryProvider(log) :
- new MappedFileMemoryProvider(
- log,
- allocPath);
-
- PageMemory pageMem = createPageMemory(memProvider, memCfg, plcCfg, memMetrics);
-
- return new MemoryPolicy(pageMem, plcCfg, memMetrics, createPageEvictionTracker(plcCfg, pageMem));
- }
-
- /**
- * @param plc Memory Policy Configuration.
- * @param pageMem Page memory.
- */
- private PageEvictionTracker createPageEvictionTracker(MemoryPolicyConfiguration plc, PageMemory pageMem) {
- if (plc.getPageEvictionMode() == DataPageEvictionMode.DISABLED)
- return new NoOpPageEvictionTracker();
-
- assert pageMem instanceof PageMemoryNoStoreImpl : pageMem.getClass();
-
- PageMemoryNoStoreImpl pageMem0 = (PageMemoryNoStoreImpl)pageMem;
-
- if (Boolean.getBoolean("override.fair.fifo.page.eviction.tracker"))
- return new FairFifoPageEvictionTracker(pageMem0, plc, cctx);
-
- switch (plc.getPageEvictionMode()) {
- case RANDOM_LRU:
- return new RandomLruPageEvictionTracker(pageMem0, plc, cctx);
- case RANDOM_2_LRU:
- return new Random2LruPageEvictionTracker(pageMem0, plc, cctx);
- default:
- return new NoOpPageEvictionTracker();
- }
- }
-
- /**
- * Builds allocation path for memory mapped file to be used with PageMemory.
- *
- * @param plc MemoryPolicyConfiguration.
- */
- @Nullable protected File buildAllocPath(MemoryPolicyConfiguration plc) {
- String path = plc.getSwapFilePath();
-
- if (path == null)
- return null;
-
- String consId = String.valueOf(cctx.discovery().consistentId());
-
- consId = consId.replaceAll("[:,\\.]", "_");
-
- return buildPath(path, consId);
- }
-
- /**
- * Creates PageMemory with given size and memory provider.
- *
- * @param memProvider Memory provider.
- * @param memCfg Memory configuartion.
- * @param memPlcCfg Memory policy configuration.
- * @param memMetrics MemoryMetrics to collect memory usage metrics.
- * @return PageMemory instance.
- */
- protected PageMemory createPageMemory(
- DirectMemoryProvider memProvider,
- MemoryConfiguration memCfg,
- MemoryPolicyConfiguration memPlcCfg,
- MemoryMetricsImpl memMetrics
- ) {
- memMetrics.persistenceEnabled(false);
-
- return new PageMemoryNoStoreImpl(
- log,
- memProvider,
- cctx,
- memCfg.getPageSize(),
- memPlcCfg,
- memMetrics,
- false
- );
- }
-
- /**
- * @param path Path to the working directory.
- * @param consId Consistent ID of the local node.
- * @return DB storage path.
- */
- protected File buildPath(String path, String consId) {
- String igniteHomeStr = U.getIgniteHome();
-
- File igniteHome = igniteHomeStr != null ? new File(igniteHomeStr) : null;
-
- File workDir = igniteHome == null ? new File(path) : new File(igniteHome, path);
-
- return new File(workDir, consId);
- }
-
- /** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
- start0();
- }
-
- /** {@inheritDoc} */
- @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
- stop0(true);
- }
-
- /**
- * @return Name of MemoryPolicyConfiguration for internal caches.
- */
- public String systemMemoryPolicyName() {
- return SYSTEM_MEMORY_POLICY_NAME;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheSnapshotManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheSnapshotManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheSnapshotManager.java
deleted file mode 100644
index 91957db..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/IgniteCacheSnapshotManager.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.nio.ByteBuffer;
-import java.util.NavigableMap;
-import java.util.UUID;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public class IgniteCacheSnapshotManager extends GridCacheSharedManagerAdapter {
- /** Snapshot started lock filename. */
- public static final String SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME = "snapshot-started.loc";
-
- /**
- * @param initiatorNodeId Initiator node id.
- * @param snapshotOperation Snapshot operation.
- */
- @Nullable public IgniteInternalFuture startLocalSnapshotOperation(
- UUID initiatorNodeId,
- SnapshotOperation snapshotOperation
- ) throws IgniteCheckedException {
- return null;
- }
-
- /**
- * @param snapOp current snapshot operation.
- *
- * @return {@code true} if next operation must be snapshot, {@code false} if checkpoint must be executed.
- */
- public boolean onMarkCheckPointBegin(
- SnapshotOperation snapOp,
- NavigableMap<T2<Integer, Integer>, T2<Integer, Integer>> map
- ) throws IgniteCheckedException {
- return false;
- }
-
- /**
- *
- */
- public void restoreState() throws IgniteCheckedException {
- // No-op.
- }
-
- /**
- *
- */
- public void onCheckPointBegin() {
- // No-op.
- }
-
- /**
- *
- */
- public void beforeCheckpointPageWritten() {
- // No-op.
- }
-
- /**
- *
- */
- public void afterCheckpointPageWritten() {
- // No-op.
- }
-
- /**
- * @param fullId Full id.
- */
- public void beforePageWrite(FullPageId fullId) {
- // No-op.
- }
-
- /**
- * @param fullId Full id.
- */
- public void onPageWrite(FullPageId fullId, ByteBuffer tmpWriteBuf) {
- // No-op.
- }
-
- /**
- * @param cctx Cctx.
- */
- public void onCacheStop(GridCacheContext cctx) {
- // No-op.
- }
-
- /**
- *
- */
- public void onChangeTrackerPage(
- Long page,
- FullPageId fullId,
- PageMemory pageMem
- ) throws IgniteCheckedException {
-
- }
-
- /**
- *
- */
- public void flushDirtyPageHandler(
- FullPageId fullId,
- ByteBuffer pageBuf,
- Integer tag
- ) throws IgniteCheckedException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java
deleted file mode 100644
index 2e5a78c..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsImpl.java
+++ /dev/null
@@ -1,284 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.MemoryMetrics;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeListImpl;
-import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jsr166.LongAdder8;
-
-/**
- *
- */
-public class MemoryMetricsImpl implements MemoryMetrics {
- /** */
- private FreeListImpl freeList;
-
- /** */
- private final LongAdder8 totalAllocatedPages = new LongAdder8();
-
- /**
- * Counter for number of pages occupied by large entries (one entry is larger than one page).
- */
- private final LongAdder8 largeEntriesPages = new LongAdder8();
-
- /** Counter for number of dirty pages. */
- private LongAdder8 dirtyPages = new LongAdder8();
-
- /** */
- private volatile boolean metricsEnabled;
-
- /** */
- private boolean persistenceEnabled;
-
- /** */
- private volatile int subInts;
-
- /** Allocation rate calculator. */
- private volatile HitRateMetrics allocRate = new HitRateMetrics(60_000, 5);
-
- /** */
- private volatile HitRateMetrics pageReplaceRate = new HitRateMetrics(60_000, 5);
-
- /** */
- private final MemoryPolicyConfiguration memPlcCfg;
-
- /** */
- private PageMemory pageMem;
-
- /** Time interval (in milliseconds) when allocations/evictions are counted to calculate rate. */
- private volatile long rateTimeInterval;
-
- /**
- * @param memPlcCfg MemoryPolicyConfiguration.
- */
- public MemoryMetricsImpl(MemoryPolicyConfiguration memPlcCfg) {
- this.memPlcCfg = memPlcCfg;
-
- metricsEnabled = memPlcCfg.isMetricsEnabled();
-
- rateTimeInterval = memPlcCfg.getRateTimeInterval();
-
- subInts = memPlcCfg.getSubIntervals();
- }
-
- /** {@inheritDoc} */
- @Override public String getName() {
- return U.maskName(memPlcCfg.getName());
- }
-
- /** {@inheritDoc} */
- @Override public long getTotalAllocatedPages() {
- return metricsEnabled ? totalAllocatedPages.longValue() : 0;
- }
-
- /** {@inheritDoc} */
- @Override public float getAllocationRate() {
- if (!metricsEnabled)
- return 0;
-
- return ((float) allocRate.getRate()) / rateTimeInterval;
- }
-
- /** {@inheritDoc} */
- @Override public float getEvictionRate() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public float getLargeEntriesPagesPercentage() {
- if (!metricsEnabled)
- return 0;
-
- return totalAllocatedPages.longValue() != 0 ?
- (float) largeEntriesPages.doubleValue() / totalAllocatedPages.longValue()
- : 0;
- }
-
- /** {@inheritDoc} */
- @Override public float getPagesFillFactor() {
- if (!metricsEnabled || freeList == null)
- return 0;
-
- return freeList.fillFactor();
- }
-
- /** {@inheritDoc} */
- @Override public long getDirtyPages() {
- if (!metricsEnabled || !persistenceEnabled)
- return 0;
-
- return dirtyPages.longValue();
- }
-
- /** {@inheritDoc} */
- @Override public float getPagesReplaceRate() {
- if (!metricsEnabled || !persistenceEnabled)
- return 0;
-
- return ((float) pageReplaceRate.getRate()) / rateTimeInterval;
- }
-
- /** {@inheritDoc} */
- @Override public long getPhysicalMemoryPages() {
- if (!metricsEnabled || !persistenceEnabled)
- return 0;
-
- assert pageMem != null;
-
- return pageMem.loadedPages();
- }
-
- /**
- * Updates pageReplaceRate metric.
- */
- public void updatePageReplaceRate() {
- if (metricsEnabled)
- pageReplaceRate.onHit();
- }
-
- /**
- * Increments dirtyPages counter.
- */
- public void incrementDirtyPages() {
- if (metricsEnabled)
- dirtyPages.increment();
- }
-
- /**
- * Decrements dirtyPages counter.
- */
- public void decrementDirtyPages() {
- if (metricsEnabled)
- dirtyPages.decrement();
- }
-
- /**
- * Resets dirtyPages counter to zero.
- */
- public void resetDirtyPages() {
- if (metricsEnabled)
- dirtyPages.reset();
- }
-
- /**
- * Increments totalAllocatedPages counter.
- */
- public void incrementTotalAllocatedPages() {
- if (metricsEnabled) {
- totalAllocatedPages.increment();
-
- updateAllocationRateMetrics();
- }
- }
-
- /**
- *
- */
- private void updateAllocationRateMetrics() {
- allocRate.onHit();
- }
-
- /**
- * @param intervalNum Interval number.
- */
- private long subInt(int intervalNum) {
- return (rateTimeInterval * intervalNum) / subInts;
- }
-
- /**
- *
- */
- public void incrementLargeEntriesPages() {
- if (metricsEnabled)
- largeEntriesPages.increment();
- }
-
- /**
- *
- */
- public void decrementLargeEntriesPages() {
- if (metricsEnabled)
- largeEntriesPages.decrement();
- }
-
- /**
- * Enable metrics.
- */
- public void enableMetrics() {
- metricsEnabled = true;
- }
-
- /**
- * Disable metrics.
- */
- public void disableMetrics() {
- metricsEnabled = false;
- }
-
- /**
- * @param persistenceEnabled Persistence enabled.
- */
- public void persistenceEnabled(boolean persistenceEnabled) {
- this.persistenceEnabled = persistenceEnabled;
- }
-
- /**
- * @param pageMem Page mem.
- */
- public void pageMemory(PageMemory pageMem) {
- this.pageMem = pageMem;
- }
-
- /**
- * @param rateTimeInterval Time interval (in milliseconds) used to calculate allocation/eviction rate.
- */
- public void rateTimeInterval(long rateTimeInterval) {
- this.rateTimeInterval = rateTimeInterval;
-
- allocRate = new HitRateMetrics((int) rateTimeInterval, subInts);
- pageReplaceRate = new HitRateMetrics((int) rateTimeInterval, subInts);
- }
-
- /**
- * Sets number of subintervals the whole rateTimeInterval will be split into to calculate allocation rate.
- *
- * @param subInts Number of subintervals.
- */
- public void subIntervals(int subInts) {
- assert subInts > 0;
-
- if (this.subInts == subInts)
- return;
-
- if (rateTimeInterval / subInts < 10)
- subInts = (int) rateTimeInterval / 10;
-
- allocRate = new HitRateMetrics((int) rateTimeInterval, subInts);
- pageReplaceRate = new HitRateMetrics((int) rateTimeInterval, subInts);
- }
-
- /**
- * @param freeList Free list.
- */
- void freeList(FreeListImpl freeList) {
- this.freeList = freeList;
- }
-}
[47/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
IGNITE-5267 - Moved ignite-ps module to ignite-core
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6bf5ce46
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6bf5ce46
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6bf5ce46
Branch: refs/heads/ignite-5267-merge-ea
Commit: 6bf5ce46c1e6c4e3dcf042ac91a5b61a726c5821
Parents: bdbba0e
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Sun Jun 11 23:02:44 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Sun Jun 11 23:02:48 2017 +0300
----------------------------------------------------------------------
.../benchmarks/jmh/tree/BPlusTreeBenchmark.java | 18 +-
modules/core/pom.xml | 14 +
.../ignite/internal/IgniteComponentType.java | 17 +-
.../apache/ignite/internal/IgniteKernal.java | 2 +-
.../pagemem/impl/PageMemoryNoStoreImpl.java | 4 +-
.../delta/DataPageInsertFragmentRecord.java | 2 +-
.../wal/record/delta/DataPageInsertRecord.java | 2 +-
.../wal/record/delta/DataPageRemoveRecord.java | 2 +-
.../delta/DataPageSetFreeListPageRecord.java | 2 +-
.../wal/record/delta/DataPageUpdateRecord.java | 2 +-
.../wal/record/delta/FixCountRecord.java | 4 +-
.../record/delta/FixLeftmostChildRecord.java | 4 +-
.../pagemem/wal/record/delta/FixRemoveId.java | 4 +-
.../wal/record/delta/InitNewPageRecord.java | 2 +-
.../pagemem/wal/record/delta/InsertRecord.java | 2 +-
.../wal/record/delta/MetaPageAddRootRecord.java | 2 +-
.../wal/record/delta/MetaPageCutRootRecord.java | 3 +-
.../wal/record/delta/MetaPageInitRecord.java | 4 +-
.../delta/MetaPageInitRootInlineRecord.java | 2 +-
.../record/delta/MetaPageInitRootRecord.java | 2 +-
.../delta/MetaPageUpdateLastAllocatedIndex.java | 4 +-
...aPageUpdateLastSuccessfulFullSnapshotId.java | 2 +-
.../MetaPageUpdateLastSuccessfulSnapshotId.java | 2 +-
.../delta/MetaPageUpdateNextSnapshotId.java | 2 +-
.../MetaPageUpdatePartitionDataRecord.java | 2 +-
.../wal/record/delta/NewRootInitRecord.java | 2 +-
.../delta/PageListMetaResetCountRecord.java | 2 +-
.../record/delta/PagesListAddPageRecord.java | 2 +-
.../delta/PagesListInitNewPageRecord.java | 4 +-
.../record/delta/PagesListRemovePageRecord.java | 2 +-
.../record/delta/PagesListSetNextRecord.java | 2 +-
.../delta/PagesListSetPreviousRecord.java | 2 +-
.../pagemem/wal/record/delta/RecycleRecord.java | 2 +-
.../pagemem/wal/record/delta/RemoveRecord.java | 4 +-
.../pagemem/wal/record/delta/ReplaceRecord.java | 2 +-
.../record/delta/SplitExistingPageRecord.java | 4 +-
.../record/delta/TrackingPageDeltaRecord.java | 2 +-
.../cache/CacheAffinitySharedManager.java | 2 +-
.../processors/cache/CacheGroupContext.java | 12 +-
.../processors/cache/GridCacheAdapter.java | 2 +-
.../processors/cache/GridCacheContext.java | 2 +-
.../processors/cache/GridCacheMapEntry.java | 6 +-
.../processors/cache/GridCacheProcessor.java | 19 +-
.../cache/GridCacheSharedContext.java | 4 +-
.../cache/IgniteCacheOffheapManager.java | 8 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 28 +-
.../cache/IgniteRebalanceIterator.java | 2 +-
.../processors/cache/database/CacheDataRow.java | 57 -
.../cache/database/CacheDataRowAdapter.java | 593 ---
.../cache/database/CacheSearchRow.java | 45 -
.../database/CheckpointLockStateChecker.java | 28 -
.../cache/database/DataStructure.java | 384 --
.../IgniteCacheDatabaseSharedManager.java | 968 ----
.../database/IgniteCacheSnapshotManager.java | 133 -
.../cache/database/MemoryMetricsImpl.java | 284 --
.../cache/database/MemoryMetricsMXBeanImpl.java | 131 -
.../cache/database/MemoryMetricsSnapshot.java | 112 -
.../processors/cache/database/MemoryPolicy.java | 83 -
.../processors/cache/database/MetaStore.java | 51 -
.../cache/database/MetadataStorage.java | 417 --
.../database/PersistenceMetricsSnapshot.java | 144 -
.../processors/cache/database/RootPage.java | 62 -
.../processors/cache/database/RowStore.java | 107 -
.../evict/FairFifoPageEvictionTracker.java | 76 -
.../database/evict/NoOpPageEvictionTracker.java | 50 -
.../evict/PageAbstractEvictionTracker.java | 171 -
.../database/evict/PageEvictionTracker.java | 52 -
.../evict/Random2LruPageEvictionTracker.java | 180 -
.../evict/RandomLruPageEvictionTracker.java | 159 -
.../cache/database/freelist/FreeList.java | 51 -
.../cache/database/freelist/FreeListImpl.java | 594 ---
.../cache/database/freelist/PagesList.java | 1484 ------
.../database/freelist/io/PagesListMetaIO.java | 186 -
.../database/freelist/io/PagesListNodeIO.java | 231 -
.../cache/database/tree/BPlusTree.java | 4808 ------------------
.../cache/database/tree/io/BPlusIO.java | 403 --
.../cache/database/tree/io/BPlusInnerIO.java | 180 -
.../cache/database/tree/io/BPlusLeafIO.java | 56 -
.../cache/database/tree/io/BPlusMetaIO.java | 182 -
.../cache/database/tree/io/CacheVersionIO.java | 184 -
.../cache/database/tree/io/DataPageIO.java | 1450 ------
.../cache/database/tree/io/DataPagePayload.java | 64 -
.../cache/database/tree/io/IOVersions.java | 99 -
.../cache/database/tree/io/PageIO.java | 548 --
.../cache/database/tree/io/PageMetaIO.java | 222 -
.../tree/io/PagePartitionCountersIO.java | 175 -
.../database/tree/io/PagePartitionMetaIO.java | 163 -
.../cache/database/tree/io/TrackingPageIO.java | 334 --
.../cache/database/tree/reuse/ReuseBag.java | 33 -
.../cache/database/tree/reuse/ReuseList.java | 43 -
.../database/tree/reuse/ReuseListImpl.java | 95 -
.../cache/database/tree/util/PageHandler.java | 511 --
.../database/tree/util/PageLockListener.java | 69 -
.../distributed/dht/GridDhtLocalPartition.java | 2 +-
.../dht/GridPartitionedGetFuture.java | 2 +-
.../dht/GridPartitionedSingleGetFuture.java | 2 +-
.../dht/atomic/GridDhtAtomicCache.java | 3 +-
.../dht/colocated/GridDhtColocatedCache.java | 3 +-
.../colocated/GridDhtDetachedCacheEntry.java | 2 +-
.../dht/preloader/GridDhtPartitionSupplier.java | 2 +-
.../distributed/near/GridNearCacheEntry.java | 2 +-
.../local/atomic/GridLocalAtomicCache.java | 3 +-
.../cache/persistence/CacheDataRow.java | 57 +
.../cache/persistence/CacheDataRowAdapter.java | 593 +++
.../cache/persistence/CacheSearchRow.java | 45 +
.../persistence/CheckpointLockStateChecker.java | 28 +
.../cache/persistence/DataStructure.java | 384 ++
.../cache/persistence/DbCheckpointListener.java | 38 +
.../FullPageIdIterableComparator.java | 51 +
.../GridCacheDatabaseSharedManager.java | 3113 ++++++++++++
.../persistence/GridCacheOffheapManager.java | 1316 +++++
.../IgniteCacheDatabaseSharedManager.java | 968 ++++
.../persistence/IgniteCacheSnapshotManager.java | 133 +
.../cache/persistence/MemoryMetricsImpl.java | 284 ++
.../persistence/MemoryMetricsMXBeanImpl.java | 131 +
.../persistence/MemoryMetricsSnapshot.java | 112 +
.../cache/persistence/MemoryPolicy.java | 83 +
.../processors/cache/persistence/MetaStore.java | 51 +
.../cache/persistence/MetadataStorage.java | 417 ++
.../persistence/PersistenceMetricsImpl.java | 297 ++
.../persistence/PersistenceMetricsSnapshot.java | 144 +
.../processors/cache/persistence/RootPage.java | 62 +
.../processors/cache/persistence/RowStore.java | 107 +
.../evict/FairFifoPageEvictionTracker.java | 76 +
.../evict/NoOpPageEvictionTracker.java | 50 +
.../evict/PageAbstractEvictionTracker.java | 171 +
.../persistence/evict/PageEvictionTracker.java | 52 +
.../evict/Random2LruPageEvictionTracker.java | 180 +
.../evict/RandomLruPageEvictionTracker.java | 159 +
.../cache/persistence/file/FilePageStore.java | 529 ++
.../persistence/file/FilePageStoreManager.java | 671 +++
.../cache/persistence/freelist/FreeList.java | 51 +
.../persistence/freelist/FreeListImpl.java | 594 +++
.../cache/persistence/freelist/PagesList.java | 1482 ++++++
.../freelist/io/PagesListMetaIO.java | 186 +
.../freelist/io/PagesListNodeIO.java | 231 +
.../pagemem/CheckpointMetricsTracker.java | 183 +
.../persistence/pagemem/EvictCandidate.java | 77 +
.../persistence/pagemem/FullPageIdTable.java | 556 ++
.../cache/persistence/pagemem/PageMemoryEx.java | 149 +
.../persistence/pagemem/PageMemoryImpl.java | 2313 +++++++++
.../cache/persistence/tree/BPlusTree.java | 4808 ++++++++++++++++++
.../cache/persistence/tree/io/BPlusIO.java | 403 ++
.../cache/persistence/tree/io/BPlusInnerIO.java | 180 +
.../cache/persistence/tree/io/BPlusLeafIO.java | 56 +
.../cache/persistence/tree/io/BPlusMetaIO.java | 182 +
.../persistence/tree/io/CacheVersionIO.java | 184 +
.../cache/persistence/tree/io/DataPageIO.java | 1450 ++++++
.../persistence/tree/io/DataPagePayload.java | 64 +
.../cache/persistence/tree/io/IOVersions.java | 99 +
.../cache/persistence/tree/io/PageIO.java | 548 ++
.../cache/persistence/tree/io/PageMetaIO.java | 222 +
.../tree/io/PagePartitionCountersIO.java | 175 +
.../tree/io/PagePartitionMetaIO.java | 163 +
.../persistence/tree/io/TrackingPageIO.java | 334 ++
.../cache/persistence/tree/reuse/ReuseBag.java | 33 +
.../cache/persistence/tree/reuse/ReuseList.java | 43 +
.../persistence/tree/reuse/ReuseListImpl.java | 95 +
.../persistence/tree/util/PageHandler.java | 511 ++
.../persistence/tree/util/PageLockListener.java | 69 +
.../wal/ByteBufferBackedDataInput.java | 40 +
.../wal/ByteBufferBackedDataInputImpl.java | 173 +
.../cache/persistence/wal/FileInput.java | 453 ++
.../cache/persistence/wal/FileWALPointer.java | 139 +
.../wal/FileWriteAheadLogManager.java | 2533 +++++++++
.../cache/persistence/wal/RecordSerializer.java | 52 +
.../persistence/wal/SegmentEofException.java | 37 +
.../IgniteDataIntegrityViolationException.java | 41 +
.../persistence/wal/crc/PureJavaCrc32.java | 655 +++
.../persistence/wal/record/HeaderRecord.java | 50 +
.../wal/serializer/RecordV1Serializer.java | 1654 ++++++
.../cache/query/GridCacheQueryManager.java | 2 +-
.../continuous/CacheContinuousQueryManager.java | 2 +-
.../cluster/GridClusterStateProcessor.java | 154 +-
.../processors/igfs/IgfsDataManager.java | 2 +-
.../schema/SchemaIndexCacheVisitorImpl.java | 2 +-
.../internal/ClusterNodeMetricsSelfTest.java | 2 +-
.../pagemem/impl/PageMemoryNoLoadSelfTest.java | 4 +-
.../processors/cache/IgniteCacheGroupsTest.java | 2 +-
.../MemoryPolicyInitializationTest.java | 313 --
.../database/tree/io/TrackingPageIOTest.java | 283 --
.../IgnitePdsAtomicCacheRebalancingTest.java | 40 +
.../IgnitePdsCacheRebalancingAbstractTest.java | 523 ++
.../IgnitePdsClientNearCachePutGetTest.java | 55 +
.../IgnitePdsContinuousRestartTest.java | 259 +
.../persistence/IgnitePdsDynamicCacheTest.java | 150 +
.../IgnitePdsMultiNodePutGetRestartTest.java | 252 +
.../persistence/IgnitePdsPageSizesTest.java | 152 +
...gnitePdsRecoveryAfterFileCorruptionTest.java | 361 ++
.../IgnitePdsRemoveDuringRebalancingTest.java | 143 +
...gnitePdsSingleNodePutGetPersistenceTest.java | 55 +
.../IgnitePdsTxCacheRebalancingTest.java | 61 +
.../IgnitePersistenceMetricsSelfTest.java | 228 +
.../IgnitePersistentStoreCacheGroupsTest.java | 508 ++
...IgnitePersistentStoreDataStructuresTest.java | 196 +
.../MemoryPolicyInitializationTest.java | 313 ++
.../db/IgnitePdsMultiNodePutGetRestartTest.java | 242 +
...PdsPageEvictionDuringPartitionClearTest.java | 174 +
.../db/IgnitePdsPageEvictionTest.java | 239 +
...tePdsRebalancingOnNotStableTopologyTest.java | 215 +
.../db/IgnitePdsTransactionsHangTest.java | 304 ++
.../db/IgnitePdsWholeClusterRestartTest.java | 165 +
.../db/file/IgnitePdsCacheIntegrationTest.java | 285 ++
.../db/file/IgnitePdsEvictionTest.java | 297 ++
.../file/IgnitePdsNoActualWalHistoryTest.java | 215 +
.../file/ignitePdsCheckpointSimulationTest.java | 1000 ++++
.../persistence/db/wal/IgnitePdsWalTlbTest.java | 134 +
.../IgniteWalDirectoriesConfigurationTest.java | 54 +
.../wal/IgniteWalHistoryReservationsTest.java | 395 ++
.../IgniteWalRecoverySeveralRestartsTest.java | 347 ++
.../db/wal/IgniteWalRecoveryTest.java | 1326 +++++
.../db/wal/WalRecoveryTxLogicalRecordsTest.java | 951 ++++
.../db/wal/crc/IgniteDataIntegrityTests.java | 137 +
.../pagemem/BPlusTreePageMemoryImplTest.java | 96 +
.../BPlusTreeReuseListPageMemoryImplTest.java | 97 +
.../pagemem/FullPageIdTableTest.java | 115 +
.../MetadataStoragePageMemoryImplTest.java | 102 +
.../persistence/pagemem/NoOpWALManager.java | 128 +
.../pagemem/PageIdDistributionTest.java | 234 +
.../pagemem/PageMemoryImplNoLoadTest.java | 98 +
.../persistence/pagemem/PageMemoryImplTest.java | 119 +
.../AbstractNodeJoinTemplate.java | 743 +++
.../IgniteChangeGlobalStateAbstractTest.java | 366 ++
.../IgniteChangeGlobalStateCacheTest.java | 168 +
...IgniteChangeGlobalStateDataStreamerTest.java | 112 +
...gniteChangeGlobalStateDataStructureTest.java | 278 +
.../IgniteChangeGlobalStateFailOverTest.java | 353 ++
.../IgniteChangeGlobalStateServiceTest.java | 102 +
.../IgniteChangeGlobalStateTest.java | 711 +++
.../IgniteStandByClusterTest.java | 213 +
.../extended/GridActivateExtensionTest.java | 239 +
.../extended/GridActivationAtomicCacheSuit.java | 79 +
.../GridActivationCacheAbstractTestSuit.java | 108 +
.../GridActivationLocalAndNearCacheSuit.java | 47 +
.../GridActivationPartitionedCacheSuit.java | 80 +
.../GridActivationReplicatedCacheSuit.java | 63 +
.../join/JoinActiveNodeToActiveCluster.java | 431 ++
.../join/JoinActiveNodeToInActiveCluster.java | 227 +
.../join/JoinInActiveNodeToActiveCluster.java | 356 ++
.../join/JoinInActiveNodeToInActiveCluster.java | 226 +
...ctiveNodeToActiveClusterWithPersistence.java | 98 +
...iveNodeToInActiveClusterWithPersistence.java | 31 +
...ctiveNodeToActiveClusterWithPersistence.java | 86 +
...iveNodeToInActiveClusterWithPersistence.java | 31 +
...gniteAbstractStandByClientReconnectTest.java | 336 ++
.../IgniteStandByClientReconnectTest.java | 283 ++
...eStandByClientReconnectToNewClusterTest.java | 289 ++
.../persistence/tree/io/TrackingPageIOTest.java | 283 ++
.../database/BPlusTreeFakeReuseSelfTest.java | 4 +-
.../database/BPlusTreeReuseSelfTest.java | 6 +-
.../processors/database/BPlusTreeSelfTest.java | 20 +-
.../database/FreeListImplSelfTest.java | 12 +-
.../database/IgniteDbAbstractTest.java | 2 +-
.../IgniteDbMemoryLeakAbstractTest.java | 2 +-
.../database/MemoryMetricsSelfTest.java | 2 +-
.../database/MetadataStorageSelfTest.java | 6 +-
.../loadtests/hashmap/GridCacheTestContext.java | 5 +-
.../testsuites/IgniteCacheTestSuite2.java | 2 +-
.../IgnitePdsOutOfMemoryTestSuite.java | 38 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 87 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 82 +
.../testsuites/IgniteStandByClusterSuite.java | 46 +
.../processors/query/h2/IgniteH2Indexing.java | 2 +-
.../query/h2/database/H2PkHashIndex.java | 2 +-
.../query/h2/database/H2RowFactory.java | 2 +-
.../processors/query/h2/database/H2Tree.java | 8 +-
.../query/h2/database/H2TreeIndex.java | 6 +-
.../query/h2/database/io/H2ExtrasInnerIO.java | 10 +-
.../query/h2/database/io/H2ExtrasLeafIO.java | 10 +-
.../query/h2/database/io/H2InnerIO.java | 8 +-
.../query/h2/database/io/H2LeafIO.java | 8 +-
.../processors/query/h2/opt/GridH2Row.java | 2 +-
...niteCacheLockPartitionOnAffinityRunTest.java | 2 +-
...dexingAndGroupPutGetPersistenceSelfTest.java | 42 +
...leNodeWithIndexingPutGetPersistenceTest.java | 55 +
...gniteDbSingleNodeWithIndexingPutGetTest.java | 1 -
.../IgnitePersistentStoreSchemaLoadTest.java | 297 ++
.../h2/database/InlineIndexHelperTest.java | 2 +-
.../IgnitePdsWithIndexingTestSuite.java | 46 +
modules/pds/pom.xml | 189 -
.../cache/database/DbCheckpointListener.java | 38 -
.../database/FullPageIdIterableComparator.java | 51 -
.../GridCacheDatabaseSharedManager.java | 3113 ------------
.../cache/database/GridCacheOffheapManager.java | 1316 -----
.../cache/database/PersistenceMetricsImpl.java | 297 --
.../cache/database/file/FilePageStore.java | 529 --
.../database/file/FilePageStoreManager.java | 671 ---
.../pagemem/CheckpointMetricsTracker.java | 183 -
.../cache/database/pagemem/EvictCandidate.java | 77 -
.../cache/database/pagemem/FullPageIdTable.java | 556 --
.../cache/database/pagemem/PageMemoryEx.java | 149 -
.../cache/database/pagemem/PageMemoryImpl.java | 2313 ---------
.../database/wal/ByteBufferBackedDataInput.java | 40 -
.../wal/ByteBufferBackedDataInputImpl.java | 173 -
.../cache/database/wal/FileInput.java | 453 --
.../cache/database/wal/FileWALPointer.java | 139 -
.../database/wal/FileWriteAheadLogManager.java | 2533 ---------
.../cache/database/wal/RecordSerializer.java | 52 -
.../cache/database/wal/SegmentEofException.java | 37 -
.../IgniteDataIntegrityViolationException.java | 41 -
.../cache/database/wal/crc/PureJavaCrc32.java | 655 ---
.../cache/database/wal/record/HeaderRecord.java | 50 -
.../wal/serializer/RecordV1Serializer.java | 1654 ------
.../resources/META-INF/classnames.properties | 28 -
.../IgnitePdsAtomicCacheRebalancingTest.java | 40 -
.../IgnitePdsCacheRebalancingAbstractTest.java | 523 --
.../IgnitePdsClientNearCachePutGetTest.java | 55 -
.../IgnitePdsContinuousRestartTest.java | 259 -
.../database/IgnitePdsDynamicCacheTest.java | 150 -
.../IgnitePdsMultiNodePutGetRestartTest.java | 252 -
.../cache/database/IgnitePdsPageSizesTest.java | 152 -
...gnitePdsRecoveryAfterFileCorruptionTest.java | 361 --
.../IgnitePdsRemoveDuringRebalancingTest.java | 143 -
...gnitePdsSingleNodePutGetPersistenceTest.java | 55 -
...dexingAndGroupPutGetPersistenceSelfTest.java | 42 -
...leNodeWithIndexingPutGetPersistenceTest.java | 55 -
.../IgnitePdsTxCacheRebalancingTest.java | 61 -
.../IgnitePersistenceMetricsSelfTest.java | 228 -
.../IgnitePersistentStoreCacheGroupsTest.java | 508 --
...IgnitePersistentStoreDataStructuresTest.java | 196 -
.../IgnitePersistentStoreSchemaLoadTest.java | 297 --
.../db/IgnitePdsMultiNodePutGetRestartTest.java | 242 -
...PdsPageEvictionDuringPartitionClearTest.java | 174 -
.../database/db/IgnitePdsPageEvictionTest.java | 239 -
...tePdsRebalancingOnNotStableTopologyTest.java | 215 -
.../db/IgnitePdsTransactionsHangTest.java | 304 --
.../db/IgnitePdsWholeClusterRestartTest.java | 165 -
.../db/file/IgnitePdsCacheIntegrationTest.java | 285 --
.../database/db/file/IgnitePdsEvictionTest.java | 297 --
.../file/IgnitePdsNoActualWalHistoryTest.java | 215 -
.../file/ignitePdsCheckpointSimulationTest.java | 1000 ----
.../database/db/wal/IgnitePdsWalTlbTest.java | 134 -
.../IgniteWalDirectoriesConfigurationTest.java | 54 -
.../wal/IgniteWalHistoryReservationsTest.java | 395 --
.../IgniteWalRecoverySeveralRestartsTest.java | 347 --
.../database/db/wal/IgniteWalRecoveryTest.java | 1288 -----
.../db/wal/WalRecoveryTxLogicalRecordsTest.java | 951 ----
.../db/wal/crc/IgniteDataIntegrityTests.java | 137 -
.../pagemem/BPlusTreePageMemoryImplTest.java | 98 -
.../BPlusTreeReuseListPageMemoryImplTest.java | 99 -
.../database/pagemem/FullPageIdTableTest.java | 115 -
.../MetadataStoragePageMemoryImplTest.java | 104 -
.../database/pagemem/NoOpPageStoreManager.java | 200 -
.../cache/database/pagemem/NoOpWALManager.java | 128 -
.../pagemem/PageIdDistributionTest.java | 234 -
.../pagemem/PageMemoryImplNoLoadTest.java | 100 -
.../database/pagemem/PageMemoryImplTest.java | 121 -
.../AbstractNodeJoinTemplate.java | 743 ---
.../IgniteChangeGlobalStateAbstractTest.java | 366 --
.../IgniteChangeGlobalStateCacheTest.java | 168 -
...IgniteChangeGlobalStateDataStreamerTest.java | 112 -
...gniteChangeGlobalStateDataStructureTest.java | 278 -
.../IgniteChangeGlobalStateFailOverTest.java | 353 --
.../IgniteChangeGlobalStateServiceTest.java | 102 -
.../IgniteChangeGlobalStateTest.java | 711 ---
.../IgniteStandByClusterTest.java | 213 -
.../extended/GridActivateExtensionTest.java | 239 -
.../extended/GridActivationAtomicCacheSuit.java | 79 -
.../GridActivationCacheAbstractTestSuit.java | 108 -
.../GridActivationLocalAndNearCacheSuit.java | 47 -
.../GridActivationPartitionedCacheSuit.java | 80 -
.../GridActivationReplicatedCacheSuit.java | 63 -
.../join/JoinActiveNodeToActiveCluster.java | 431 --
.../join/JoinActiveNodeToInActiveCluster.java | 227 -
.../join/JoinInActiveNodeToActiveCluster.java | 356 --
.../join/JoinInActiveNodeToInActiveCluster.java | 226 -
...ctiveNodeToActiveClusterWithPersistence.java | 97 -
...iveNodeToInActiveClusterWithPersistence.java | 31 -
...ctiveNodeToActiveClusterWithPersistence.java | 85 -
...iveNodeToInActiveClusterWithPersistence.java | 31 -
...gniteAbstractStandByClientReconnectTest.java | 336 --
.../IgniteStandByClientReconnectTest.java | 283 --
...eStandByClientReconnectToNewClusterTest.java | 289 --
.../IgnitePdsOutOfMemoryTestSuite.java | 38 -
.../ignite/testsuites/IgnitePdsTestSuite.java | 95 -
.../ignite/testsuites/IgnitePdsTestSuite2.java | 84 -
.../testsuites/IgniteStandByClusterSuite.java | 46 -
parent/pom.xml | 1 +
pom.xml | 1 -
379 files changed, 49654 insertions(+), 49971 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
----------------------------------------------------------------------
diff --git a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
index c9ad0cf..94abc86 100644
--- a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
+++ b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/tree/BPlusTreeBenchmark.java
@@ -30,15 +30,15 @@ import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.logger.java.JavaLogger;
import org.openjdk.jmh.annotations.Benchmark;
import org.openjdk.jmh.annotations.Scope;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/pom.xml
----------------------------------------------------------------------
diff --git a/modules/core/pom.xml b/modules/core/pom.xml
index 6c714d0f..17e850d 100644
--- a/modules/core/pom.xml
+++ b/modules/core/pom.xml
@@ -211,6 +211,20 @@
<version>1.1</version>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ <version>${javassist.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
index 1b70ab4..0cd2fc1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteComponentType.java
@@ -89,22 +89,7 @@ public enum IgniteComponentType {
"org.apache.ignite.internal.processors.schedule.IgniteNoopScheduleProcessor",
"org.apache.ignite.internal.processors.schedule.IgniteScheduleProcessor",
"ignite-schedule"
- ),
-
- /** Database manager used when persistence is enabled. */
- DATABASE_MANAGER(null,
- "org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager",
- "ignite-pds"),
-
- /** Page store manager used when persistence is enabled. */
- PAGE_STORE_MANAGER(null,
- "org.apache.ignite.internal.processors.cache.database.file.FilePageStoreManager",
- "ignite-pds"),
-
- /** Write-Ahead Log manager used when persistence is enabled. */
- WAL_MANAGER(null,
- "org.apache.ignite.internal.processors.cache.database.wal.FileWriteAheadLogManager",
- "ignite-pds");
+ );
/** No-op class name. */
private final String noOpClsName;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index fa605d6..a6ad89e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -112,7 +112,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheUtilityKey;
import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
import org.apache.ignite.internal.processors.cache.IgniteInternalCache;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
import org.apache.ignite.internal.processors.closure.GridClosureProcessor;
import org.apache.ignite.internal.processors.cluster.ClusterProcessor;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
index dc72482..8f146dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/impl/PageMemoryNoStoreImpl.java
@@ -34,8 +34,8 @@ import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.GridUnsafe;
import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.OffheapReadWriteLock;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
index eeaabd1..042fbe4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertFragmentRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
/**
* Insert fragment to data page record.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
index f23d57a..17425fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageInsertRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
/**
* Insert into data page.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
index 17c7fe8..3aabe08 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageRemoveRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
index c835052..82281de 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageSetFreeListPageRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
import static org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType.DATA_PAGE_SET_FREE_LIST_PAGE;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
index 65b7172..d4e2811 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/DataPageUpdateRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
/**
* Update existing record in data page.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java
index c727710..3ab1c4a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixCountRecord.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
/**
* Fix elements count record.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java
index 94155c9..15ee7dd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixLeftmostChildRecord.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
/**
* Fix leftmost child.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java
index b9900e1..b6ea410 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/FixRemoveId.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
/**
* Fix remove ID record.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
index f5607dd..f883e06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java
index f6d8f8c..d0e66a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InsertRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java
index 176df33..1f1e976 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageAddRootRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
/**
* New root in meta page.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java
index 50ccddc..3fc42a1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageCutRootRecord.java
@@ -17,10 +17,9 @@
package org.apache.ignite.internal.pagemem.wal.record.delta;
-import java.nio.ByteBuffer;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java
index 1daf5a1..692b8f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRecord.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java
index 212f5b9..7eb8426 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootInlineRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java
index 280310f..0eb61b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageInitRootRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
/**
* Initialize new meta page.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
index d916be2..548735e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastAllocatedIndex.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java
index e322b4f..34be353 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulFullSnapshotId.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java
index df9d778..7fca5dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateLastSuccessfulSnapshotId.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
index 1403bd7..0cd72ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdateNextSnapshotId.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageMetaIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
index b28dd52..15d1a0c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/MetaPageUpdatePartitionDataRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java
index 277d773..bdfdef5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/NewRootInitRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
/**
* Initialize new root page.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java
index 16587d3..4756c9a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PageListMetaResetCountRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
/**
* Delta record for page-list meta count reset
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java
index a503b46..2577ec3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListAddPageRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java
index b035323..673d33c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListInitNewPageRecord.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java
index 26f832c..2341129 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListRemovePageRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java
index c0bed60..30600e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetNextRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java
index 21c3ef2..590643e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/PagesListSetPreviousRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java
index 1737e12..92ea88e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RecycleRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
/**
* Recycle index page.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java
index 6d278e9..ac5d442 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/RemoveRecord.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.typedef.internal.S;
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java
index 4e85c43..2f504ad 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/ReplaceRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
/**
* Replace.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java
index 418d28b..5cb3023 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/SplitExistingPageRecord.java
@@ -19,8 +19,8 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java
index 7cd0948..05e96ff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/TrackingPageDeltaRecord.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
/**
* Delta record for updates in tracking pages
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index 70bf117..6340360 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -387,7 +387,7 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
nearCfg = req.nearCacheConfiguration();
}
else // Only static cache configured on client must be started.
- startCache = cctx.kernalContext().state().isLocalConfigure(req.cacheName());
+ startCache = cctx.kernalContext().state().isLocallyConfigured(req.cacheName());
}
else if (cctx.localNodeId().equals(req.initiatingNodeId())) {
startCache = true;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
index b85c41d..7367374 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheGroupContext.java
@@ -37,9 +37,10 @@ import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.affinity.AffinityAssignment;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentRequest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtAffinityAssignmentResponse;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
@@ -855,10 +856,7 @@ public class CacheGroupContext {
ClassLoader clsLdr = U.gridClassLoader();
try {
- offheapMgr = (IgniteCacheOffheapManager) clsLdr
- .loadClass("org.apache.ignite.internal.processors.cache.database.GridCacheOffheapManager")
- .getConstructor()
- .newInstance();
+ offheapMgr = new GridCacheOffheapManager();
}
catch (Exception e) {
throw new IgniteCheckedException("Failed to initialize offheap manager", e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 535cfdd..81cb1f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -83,7 +83,7 @@ import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException
import org.apache.ignite.internal.cluster.IgniteClusterEx;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.affinity.GridCacheAffinityImpl;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.IgniteExternalizableExpiryPolicy;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
index 44cf4e0..83f7aa0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheContext.java
@@ -56,7 +56,7 @@ import org.apache.ignite.internal.managers.deployment.GridDeploymentManager;
import org.apache.ignite.internal.managers.discovery.GridDiscoveryManager;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 28656fb..eea8935 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -38,9 +38,9 @@ import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.GridCacheUpdateAtomicResult.UpdateOutcome;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRowAdapter;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridDhtAtomicAbstractUpdateFuture;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
index 5ee80ef..55b4998 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java
@@ -77,11 +77,13 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheJoinNodeDiscoveryData.CacheInfo;
import org.apache.ignite.internal.processors.affinity.GridAffinityAssignmentCache;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheSnapshotManager;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCache;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheAdapter;
@@ -95,6 +97,7 @@ import org.apache.ignite.internal.processors.cache.dr.GridCacheDrManager;
import org.apache.ignite.internal.processors.cache.jta.CacheJtaManagerAdapter;
import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
import org.apache.ignite.internal.processors.cache.local.atomic.GridLocalAtomicCache;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.query.GridCacheDistributedQueryManager;
import org.apache.ignite.internal.processors.cache.query.GridCacheLocalQueryManager;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
@@ -2175,11 +2178,11 @@ public class GridCacheProcessor extends GridProcessorAdapter {
" configuration will be ignored).");
}
- dbMgr = IgniteComponentType.DATABASE_MANAGER.create(ctx, false);
+ dbMgr = new GridCacheDatabaseSharedManager(ctx);
- pageStoreMgr = IgniteComponentType.PAGE_STORE_MANAGER.create(ctx, false);
+ pageStoreMgr = new FilePageStoreManager(ctx);
- walMgr = IgniteComponentType.WAL_MANAGER.create(ctx, false);
+ walMgr = new FileWriteAheadLogManager(ctx);
}
else
dbMgr = new IgniteCacheDatabaseSharedManager();
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
index be1ab3d..0ee0020 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheSharedContext.java
@@ -40,8 +40,8 @@ import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheSnapshotManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionExchangeId;
import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index 8951396..001848e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -21,10 +21,10 @@ import java.util.Map;
import javax.cache.Cache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.RootPage;
-import org.apache.ignite.internal.processors.cache.database.RowStore;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.RowStore;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.GridAtomicLong;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index ae47443..b9afad8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -36,20 +36,20 @@ import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRowAdapter;
-import org.apache.ignite.internal.processors.cache.database.CacheSearchRow;
-import org.apache.ignite.internal.processors.cache.database.RootPage;
-import org.apache.ignite.internal.processors.cache.database.RowStore;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPagePayload;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.RowStore;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryManager;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteRebalanceIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteRebalanceIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteRebalanceIterator.java
index c191c72..4efe640 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteRebalanceIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteRebalanceIterator.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.cache;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.util.lang.GridCloseableIterator;
/**
[20/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
new file mode 100644
index 0000000..76a5260
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/AbstractNodeJoinTemplate.java
@@ -0,0 +1,743 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+import static org.apache.ignite.internal.util.IgniteUtils.field;
+
+/**
+ *
+ */
+public abstract class AbstractNodeJoinTemplate extends GridCommonAbstractTest {
+ /** Cache 1. */
+ protected static final String cache1 = "cache1";
+
+ /** Cache 2. */
+ protected static final String cache2 = "cache2";
+
+ //Todo Cache with node filter.
+ protected static final String cache3 = "cache3";
+
+ protected static final String cache4 = "cache4";
+
+ protected static final String cache5 = "cache5";
+
+ /** Caches info. */
+ public static final String CACHES_INFO = "cachesInfo";
+
+ /** Registered caches. */
+ public static final String REGISTERED_CACHES = "registeredCaches";
+
+ /** Caches. */
+ public static final String CACHES = "caches";
+
+ /**
+ * @param ig Ig.
+ */
+ protected static Map<String, DynamicCacheDescriptor> cacheDescriptors(IgniteEx ig) {
+ return field(field(ig.context().cache(), CACHES_INFO), REGISTERED_CACHES);
+ }
+
+ /**
+ * @param ig Ig.
+ */
+ protected static Map<String, GridCacheAdapter> caches(IgniteEx ig){
+ return field(ig.context().cache(), CACHES);
+ }
+
+ /**
+ *
+ */
+ public abstract JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception;
+
+ /**
+ *
+ */
+ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception;
+
+ /**
+ *
+ */
+ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception;
+
+ /**
+ *
+ */
+ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception;
+
+ /**
+ *
+ */
+ public abstract JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception;
+
+ // Client node join.
+
+ public abstract JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception;
+
+ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception;
+
+ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception;
+
+ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception;
+
+ public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testJoinWithOutConfiguration() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testStaticCacheConfigurationOnJoin() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testStaticCacheConfigurationInCluster() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testStaticCacheConfigurationSameOnBoth() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testStaticCacheConfigurationDifferentOnBoth() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testJoinClientWithOutConfiguration() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testJoinClientStaticCacheConfigurationOnJoin() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testJoinClientStaticCacheConfigurationInCluster() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception;
+
+ /**
+ *
+ */
+ public abstract void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception;
+
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @param idx Index.
+ */
+ protected String name(int idx) {
+ return getTestIgniteInstanceName(idx);
+ }
+
+ /**
+ * @param name Name.
+ */
+ protected IgniteConfiguration cfg(String name) throws Exception {
+ try {
+ return getConfiguration(name);
+ }
+ catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ *
+ */
+ protected JoinNodeTestPlanBuilder builder() {
+ return JoinNodeTestPlanBuilder.builder();
+ }
+
+ /**
+ * @param cfgs Cfgs.
+ */
+ protected static <T> T[] buildConfiguration(T... cfgs) {
+ return cfgs;
+ }
+
+ /**
+ *
+ */
+ protected CacheConfiguration atomicCfg() {
+ return new CacheConfiguration(cache1)
+ .setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ }
+
+ /**
+ *
+ */
+ protected CacheConfiguration transactionCfg() {
+ return new CacheConfiguration(cache2)
+ .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ }
+
+ /**
+ *
+ */
+ protected CacheConfiguration[] allCacheConfigurations() {
+ return buildConfiguration(atomicCfg(), transactionCfg());
+ }
+
+ /** Set client. */
+ protected final IgniteClosure<IgniteConfiguration, IgniteConfiguration> setClient =
+ new IgniteClosure<IgniteConfiguration, IgniteConfiguration>() {
+ @Override public IgniteConfiguration apply(IgniteConfiguration cfg) {
+ return cfg.setClientMode(true);
+ }
+ };
+
+ /** Ip finder. */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ return super.getConfiguration(name)
+ .setDiscoverySpi(
+ new TcpDiscoverySpi()
+ .setIpFinder(ipFinder)
+ );
+ }
+
+ /** {@inheritDoc} */
+ protected IgniteConfiguration persistentCfg(IgniteConfiguration cfg) throws Exception {
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+ return cfg;
+ }
+
+ /**
+ *
+ */
+ public static class JoinNodeTestPlanBuilder extends GridCommonAbstractTest {
+ /** String plan builder. */
+ private final StringBuilder strPlanBuilder = new StringBuilder().append("**** Execution plan ****\n");
+
+ /** Nodes. */
+ protected List<String> nodes = new ArrayList<>(4);
+
+ /** Cluster config. */
+ private IgniteConfiguration[] clusterCfg;
+
+ /** Node config. */
+ private IgniteConfiguration nodeCfg;
+
+ /** State default. */
+ private static final Boolean stateDefault = new Boolean(true);
+
+ /** State. */
+ private Boolean state = stateDefault;
+
+ /** Noop. */
+ private static final Runnable Noop = new Runnable() {
+ @Override public void run() {
+ }
+ };
+
+ /** After cluster started. */
+ private Runnable afterClusterStarted = Noop;
+
+ /** After node join. */
+ private Runnable afterNodeJoin = Noop;
+
+ /** After activate. */
+ private Runnable afterActivate = Noop;
+
+ /** After de activate. */
+ private Runnable afterDeActivate = Noop;
+
+ private IgniteCallable<List<CacheConfiguration>> dynamicCacheStart =
+ new IgniteCallable<List<CacheConfiguration>>() {
+ @Override public List<CacheConfiguration> call() throws Exception {
+ return Arrays.asList(new CacheConfiguration(cache4), new CacheConfiguration(cache5));
+ }
+ };
+
+ private IgniteCallable<List<String>> dynamicCacheStop =
+ new IgniteCallable<List<String>>() {
+ @Override public List<String> call() throws Exception {
+ return Arrays.asList(cache4, cache5);
+ }
+ };
+
+ private Runnable afterDynamicCacheStarted = Noop;
+
+ private Runnable afterDynamicCacheStopped = Noop;
+
+ /** End. */
+ private Runnable end = Noop;
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder clusterConfiguration(IgniteConfiguration... cfgs) throws Exception {
+ clusterCfg = cfgs;
+
+ strPlanBuilder.append("Start cluster:\n");
+
+ for (IgniteConfiguration cfg : cfgs) {
+ strPlanBuilder.append("node: ")
+ .append(cfg.getIgniteInstanceName())
+ .append(" activeOnStart - ")
+ .append(cfg.isActiveOnStart())
+ .append("\n");
+
+ CacheConfiguration[] ccfgs = cfg.getCacheConfiguration();
+
+ if (ccfgs != null) {
+ for (CacheConfiguration ccfg : ccfgs)
+ strPlanBuilder.append(" cache - ")
+ .append(ccfg.getName())
+ .append("\n");
+ }
+ }
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder nodeConfiguration(IgniteConfiguration cfg) {
+ nodeCfg = cfg;
+
+ strPlanBuilder.append("Join node:\n")
+ .append(cfg.getIgniteInstanceName())
+ .append(cfg.isClientMode() != null && cfg.isClientMode() ? " (client)" : "")
+ .append(" activeOnStart - ")
+ .append(cfg.isActiveOnStart())
+ .append("\n");
+
+ CacheConfiguration[] ccfgs = cfg.getCacheConfiguration();
+
+ if (ccfgs != null)
+ for (CacheConfiguration ccfg : ccfgs)
+ strPlanBuilder.append(" cache - ").append(ccfg.getName()).append("\n");
+
+ return this;
+ }
+
+ /**
+ * @param func Func.
+ */
+ public JoinNodeTestPlanBuilder nodeConfiguration(
+ IgniteClosure<IgniteConfiguration, IgniteConfiguration> func
+ ) {
+
+ nodeCfg = func.apply(nodeCfg);
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder afterClusterStarted(Runnable r) {
+ strPlanBuilder.append("Check after cluster start\n");
+
+ afterClusterStarted = r;
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder afterNodeJoin(Runnable r) {
+ strPlanBuilder.append("Check after node join")
+ .append("\n");
+
+ afterNodeJoin = r;
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder stateAfterJoin(boolean state) {
+ strPlanBuilder.append("Check state on all nodes after join, must be ")
+ .append(state ? "<<active>>" : "<<inactive>>")
+ .append(" \n");
+
+ this.state = state;
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder afterActivate(Runnable r) {
+ strPlanBuilder.append("Check after activate")
+ .append("\n");
+
+ afterActivate = r;
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public JoinNodeTestPlanBuilder afterDeActivate(Runnable r) {
+ strPlanBuilder.append("Check after deActivate")
+ .append("\n");
+
+ afterDeActivate = r;
+
+ return this;
+ }
+
+ public JoinNodeTestPlanBuilder dynamicCacheStart(IgniteCallable<List<CacheConfiguration>> caches){
+ strPlanBuilder.append("Dynamic caches start")
+ .append("\n");
+
+ dynamicCacheStart = caches;
+
+ return this;
+ }
+
+ public JoinNodeTestPlanBuilder afterDynamicCacheStarted(Runnable r){
+ strPlanBuilder.append("Check after dynamic caches start")
+ .append("\n");
+
+ afterDynamicCacheStarted = r;
+
+ return this;
+ }
+
+ public JoinNodeTestPlanBuilder dynamicCacheStop(IgniteCallable<List<String>> caches){
+ strPlanBuilder.append("Dynamic caches stop")
+ .append("\n");
+
+ dynamicCacheStop = caches;
+
+ return this;
+ }
+
+ public JoinNodeTestPlanBuilder afterDynamicCacheStopped(Runnable r){
+ strPlanBuilder.append("Check after dynamic caches stop")
+ .append("\n");
+
+ afterDynamicCacheStopped = r;
+
+ return this;
+ }
+
+ /**
+ * @param end End.
+ */
+ public JoinNodeTestPlanBuilder setEnd(Runnable end) {
+ strPlanBuilder.append("Check before stop")
+ .append("\n");
+
+ this.end = end;
+
+ return this;
+ }
+
+ /**
+ *
+ */
+ public void execute() throws Exception {
+ try {
+ if (state == stateDefault)
+ fail("State after join must be specific. See JoinNodeTestPlanBuilder.stateAfterJoin(boolean).");
+
+ System.out.println(strPlanBuilder.append("********************").toString());
+
+ IgniteConfiguration[] cfgs = clusterCfg;
+
+ System.out.println(">>> Start cluster");
+
+ for (IgniteConfiguration cfg : cfgs) {
+ startGrid(cfg);
+
+ nodes.add(cfg.getIgniteInstanceName());
+ }
+
+ System.out.println(">>> Check after cluster started");
+
+ afterClusterStarted.run();
+
+ System.out.println(">>> Start new node");
+
+ startGrid(nodeCfg);
+
+ nodes.add(nodeCfg.getIgniteInstanceName());
+
+ System.out.println(">>> Check after new node join in cluster");
+
+ afterNodeJoin.run();
+
+ System.out.println(">>> Check cluster state on all nodes");
+
+ IgniteEx crd = grid(nodes.get(0));
+
+ for (IgniteEx ig : grids())
+ assertEquals((boolean)state, ig.active());
+
+ if (!state) {
+ System.out.println(">>> Activate cluster");
+
+ crd.active(true);
+
+ System.out.println(">>> Check after cluster activated");
+
+ afterActivate.run();
+ }
+ else {
+ System.out.println(">>> DeActivate cluster");
+
+ crd.active(false);
+
+ System.out.println(">>> Check after cluster deActivated");
+
+ afterDeActivate.run();
+
+ System.out.println(">>> Activate cluster");
+
+ crd.active(true);
+ }
+
+ AffinityTopologyVersion next0Ver = nextMinorVersion(crd);
+
+ crd.createCaches(dynamicCacheStart.call());
+
+ awaitTopologyVersion(next0Ver);
+
+ afterDynamicCacheStarted.run();
+
+ onAllNode(new CI1<IgniteEx>() {
+ @Override public void apply(IgniteEx ig) {
+ if (ig.context().discovery().localNode().isClient())
+ return;
+
+ Assert.assertNotNull(ig.context().cache().cache(cache4));
+ Assert.assertNotNull(ig.context().cache().cache(cache5));
+
+ }
+ });
+
+ AffinityTopologyVersion next1Ver = nextMinorVersion(crd);
+
+ crd.destroyCaches(dynamicCacheStop.call());
+
+ afterDynamicCacheStopped.run();
+
+ awaitTopologyVersion(next1Ver);
+
+ onAllNode(new CI1<IgniteEx>() {
+ @Override public void apply(IgniteEx ig) {
+ if (ig.context().discovery().localNode().isClient())
+ return;
+
+ Assert.assertNull(ig.context().cache().cache(cache4));
+ Assert.assertNull(ig.context().cache().cache(cache5));
+
+ }
+ });
+
+ System.out.println(">>> Finish check");
+
+ end.run();
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ private AffinityTopologyVersion nextMinorVersion(IgniteEx ig){
+ AffinityTopologyVersion cur = ig.context().discovery().topologyVersionEx();
+
+ return new AffinityTopologyVersion(cur.topologyVersion(), cur.minorTopologyVersion() + 1);
+ }
+
+ private void awaitTopologyVersion(final AffinityTopologyVersion ver){
+ onAllNode(new CI1<IgniteEx>() {
+ @Override public void apply(IgniteEx ig) {
+ while (true) {
+ AffinityTopologyVersion locTopVer = ig.context().cache().context()
+ .exchange().readyAffinityVersion();
+
+ if (locTopVer.compareTo(ver) < 0){
+ System.out.println("Top ready " + locTopVer + " on " + ig.localNode().id());
+
+ try {
+ Thread.sleep(100);
+ }
+ catch (InterruptedException e) {
+ break;
+ }
+ }
+ else
+ break;
+ }
+ }
+ }).run();
+
+ }
+
+ /**
+ *
+ */
+ protected List<IgniteEx> grids() {
+ List<IgniteEx> res = new ArrayList<>();
+
+ for (String name : nodes)
+ res.add(grid(name));
+
+ return res;
+ }
+
+ /**
+ *
+ */
+ public static JoinNodeTestPlanBuilder builder() {
+ return new JoinNodeTestPlanBuilder();
+ }
+
+ /**
+ *
+ */
+ public Runnable checkCacheOnlySystem() {
+ return onAllNode(new IgniteInClosure<IgniteEx>() {
+ @Override public void apply(IgniteEx ig) {
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(2, desc.size());
+
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(2, caches.size());
+ }
+ });
+ }
+
+ /**
+ *
+ */
+ public Runnable checkCacheEmpty() {
+ return onAllNode(new IgniteInClosure<IgniteEx>() {
+ @Override public void apply(IgniteEx ig) {
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertTrue(desc.isEmpty());
+
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(0, caches.size());
+ }
+ });
+ }
+
+ /**
+ *
+ */
+ public Runnable checkCacheNotEmpty() {
+ return onAllNode(new IgniteInClosure<IgniteEx>() {
+ @Override public void apply(IgniteEx ig) {
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ });
+ }
+
+ /**
+ * @param cls Closure.
+ */
+ private Runnable onAllNode(final IgniteInClosure<IgniteEx> cls) {
+ return new Runnable() {
+ @Override public void run() {
+ for (IgniteEx ig : grids()) {
+ try {
+ cls.apply(ig);
+ }
+ catch (AssertionError e) {
+ System.out.println("Assertion on " + ig.name());
+
+ throw e;
+ }
+ }
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
new file mode 100644
index 0000000..d16b845
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateAbstractTest.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbstractTest {
+ /** Primary suffix. */
+ private static final String primarySuffix = "-primary";
+
+ /** BackUp suffix. */
+ private static final String backUpSuffix = "-backUp";
+
+ /** BackUp suffix. */
+ private static final String clientSuffix = "-client";
+
+ /** Primary ip finder. */
+ protected final TcpDiscoveryIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Back up ip finder. */
+ protected final TcpDiscoveryIpFinder backUpIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Consistent id count. */
+ private int consistentIdCnt;
+
+ /** Nodes. */
+ protected Map<String, Ignite> nodes = new ConcurrentHashMap<>();
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ nodes.clear();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName(), true));
+
+ startPrimaryNodes(primaryNodes());
+
+ startPrimaryClientNodes(primaryClientNodes());
+
+ startBackUpNodes(backUpNodes());
+
+ startBackUpClientNodes(backUpClientNodes());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAll(clientSuffix);
+
+ stopAll(primarySuffix);
+
+ stopAll(backUpSuffix);
+
+ nodes.clear();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName(), true));
+ }
+
+ /**
+ *
+ */
+ protected int primaryNodes() {
+ return 3;
+ }
+
+ /**
+ *
+ */
+ protected int primaryClientNodes() {
+ return 3;
+ }
+
+ /**
+ *
+ */
+ protected int backUpNodes() {
+ return 3;
+ }
+
+ /**
+ *
+ */
+ protected int backUpClientNodes() {
+ return 3;
+ }
+
+ /**
+ * @param idx idx.
+ */
+ protected Ignite primary(int idx) {
+ return nodes.get("node" + idx + primarySuffix);
+ }
+
+ /**
+ * @param idx idx.
+ */
+ protected Ignite primaryClient(int idx) {
+ return nodes.get("node" + idx + primarySuffix + clientSuffix);
+ }
+
+ /**
+ * @param idx idx.
+ */
+ protected Ignite backUp(int idx) {
+ return nodes.get("node" + idx + backUpSuffix);
+ }
+
+ /**
+ * @param idx idx.
+ */
+ protected Ignite backUpClient(int idx) {
+ return nodes.get("node" + idx + backUpSuffix + clientSuffix);
+ }
+
+ /**
+ * @param cnt Count.
+ */
+ protected void startPrimaryNodes(int cnt) throws Exception {
+ for (int i = 0; i < cnt; i++)
+ startPrimary(i);
+ }
+
+ /**
+ * @param idx Index.
+ */
+ protected void startPrimary(int idx) throws Exception {
+ String node = "node" + idx;
+
+ String name = node + primarySuffix;
+
+ IgniteConfiguration cfg = getConfiguration(name);
+ cfg.setConsistentId(node);
+ cfg.setActiveOnStart(true);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder);
+
+ Ignite ig = startGrid(name, cfg);
+
+ nodes.put(name, ig);
+ }
+
+ /**
+ * @param cnt Count.
+ */
+ protected void startBackUpNodes(int cnt) throws Exception {
+ for (int i = 0; i < cnt; i++)
+ startBackUp(i);
+ }
+
+ /**
+ * @param idx Index.
+ */
+ protected void startBackUp(int idx) throws Exception {
+ String node = "node" + idx;
+
+ String name = node + backUpSuffix;
+
+ IgniteConfiguration cfg = getConfiguration(name);
+ cfg.setConsistentId(node);
+ cfg.setActiveOnStart(false);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
+
+ Ignite ig = startGrid(name, cfg);
+
+ nodes.put(name, ig);
+ }
+
+ /**
+ * @param cnt Count.
+ */
+ protected void startPrimaryClientNodes(int cnt) throws Exception {
+ for (int i = 0; i < cnt; i++) {
+ String node = "node" + i;
+
+ String name = node + primarySuffix + clientSuffix;
+
+ IgniteConfiguration cfg = getConfiguration(name);
+ cfg.setConsistentId(node);
+ cfg.setClientMode(true);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder);
+
+ Ignite ig = startGrid(name, cfg);
+
+ nodes.put(name, ig);
+ }
+ }
+
+ /**
+ * @param cnt Count.
+ */
+ protected void startBackUpClientNodes(int cnt) throws Exception {
+ for (int i = 0; i < cnt; i++) {
+ String node = "node" + i;
+
+ String name = node + backUpSuffix + clientSuffix;
+
+ IgniteConfiguration cfg = getConfiguration(name);
+ cfg.setConsistentId(node);
+ cfg.setActiveOnStart(false);
+ cfg.setClientMode(true);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
+
+ Ignite ig = startGrid(name, cfg);
+
+ nodes.put(name, ig);
+ }
+ }
+
+ /**
+ *
+ */
+ protected Iterable<Ignite> allBackUpNodes(){
+ List<Ignite> r = new ArrayList<>();
+
+ for (String name : this.nodes.keySet())
+ if (name.contains(backUpSuffix))
+ r.add(nodes.get(name));
+
+ return r;
+ }
+
+ /**
+ *
+ */
+ protected Ignite randomBackUp(boolean includeClient) {
+ int nodes = 0;
+
+ List<Ignite> igs = new ArrayList<>();
+
+ for (String name : this.nodes.keySet())
+ if (name.contains(backUpSuffix)){
+ if (includeClient)
+ igs.add(this.nodes.get(name));
+ else {
+ if (name.contains(clientSuffix))
+ continue;
+
+ igs.add(this.nodes.get(name));
+ }
+ }
+
+ int idx = ThreadLocalRandom.current().nextInt(0, igs.size());
+
+ return igs.get(idx);
+ }
+
+
+ /**
+ * @param i Idx.
+ */
+ protected void stopPrimary(int i) {
+ String name = "node" + i + primarySuffix;
+
+ nodes.get(name).close();
+
+ nodes.remove(name);
+ }
+
+ /**
+ *
+ */
+ protected void stopAllPrimary() {
+ stopAll(primarySuffix);
+ }
+
+ /**
+ *
+ */
+ protected void stopAllBackUp() {
+ stopAll(backUpSuffix);
+ }
+
+ /**
+ *
+ */
+ protected void stopAllClient() {
+ stopAll(clientSuffix);
+ }
+
+ /**
+ * @param suffix Suffix.
+ */
+ private void stopAll(String suffix) {
+ for (String name : nodes.keySet())
+ if (name.contains(suffix)) {
+ Ignite ig = nodes.get(name);
+
+ stopGrid(ig.name());
+
+ nodes.remove(name);
+ }
+ }
+
+ /**
+ * @param gridName Grid name.
+ */
+ @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ pCfg.setPersistentStorePath(testName() + "/db");
+ pCfg.setWalArchivePath(testName() + "/db/wal/archive");
+ pCfg.setWalStorePath(testName() + "/db/wal");
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ final MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ memCfg.setPageSize(1024);
+ memCfg.setConcurrencyLevel(64);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+ memPlcCfg.setInitialSize(20 * 1024 * 1024);
+ memPlcCfg.setMaxSize(20 * 1024 * 1024);
+ memPlcCfg.setName("dfltMemPlc");
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /**
+ *
+ */
+ protected String testName() {
+ return getClass().getSimpleName();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java
new file mode 100644
index 0000000..1dc99f9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateCacheTest.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import javax.cache.configuration.Configuration;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.util.typedef.F;
+
+/**
+ *
+ */
+public class IgniteChangeGlobalStateCacheTest extends IgniteChangeGlobalStateAbstractTest {
+ /**
+ *
+ */
+ public void testCheckValueAfterActivation(){
+ String cacheName = "my-cache";
+
+ Ignite ig1P = primary(0);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ IgniteCache<String, String> cacheP = ig1P.getOrCreateCache(cacheName);
+
+ cacheP.put("key","value");
+
+ stopAllPrimary();
+
+ ig1B.active(true);
+
+ IgniteCache<String, String> cache1B = ig1B.cache(cacheName);
+ IgniteCache<String, String> cache2B = ig2B.cache(cacheName);
+ IgniteCache<String, String> cache3B = ig3B.cache(cacheName);
+
+ assertTrue(cache1B != null);
+ assertTrue(cache2B != null);
+ assertTrue(cache3B != null);
+
+ assertEquals(cache1B.get("key"), "value");
+ assertEquals(cache2B.get("key"), "value");
+ assertEquals(cache3B.get("key"), "value");
+ }
+
+ /**
+ *
+ */
+ public void testMoreKeyValueAfterActivate() throws Exception {
+ String cacheName = "my-cache";
+
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ CacheConfiguration<String, String> cacheCfg = new CacheConfiguration<>(cacheName);
+
+ IgniteCache<String, String> cache1P = ig1P.getOrCreateCache(cacheCfg);
+
+ for (int i = 0; i < 4_000; i++)
+ cache1P.put("key" + i, "value" + i);
+
+ IgniteCache<String, String> cache2P = ig2P.cache(cacheName);
+
+ for (int i = 4_000; i < 8_000; i++)
+ cache2P.put("key" + i, "value" + i);
+
+ IgniteCache<String, String> cache3P = ig3P.cache(cacheName);
+
+ for (int i = 8_000; i < 12_000; i++)
+ cache3P.put("key" + i, "value" + i);
+
+ stopAllPrimary();
+
+ ig1B.active(true);
+
+ IgniteCache<String, String> cache1B = ig1B.cache(cacheName);
+ IgniteCache<String, String> cache2B = ig2B.cache(cacheName);
+ IgniteCache<String, String> cache3B = ig3B.cache(cacheName);
+
+ assertTrue(cache1B != null);
+ assertTrue(cache2B != null);
+ assertTrue(cache3B != null);
+
+ for (int i = 0; i < 4_000; i++)
+ assertEquals("value" + i, cache1B.get("key" + i));
+
+ for (int i = 4_000; i < 8_000; i++)
+ assertEquals("value" + i, cache2B.get("key" + i));
+
+ for (int i = 8_000; i < 12_000; i++)
+ assertEquals("value" + i, cache3B.get("key" + i));
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testDeActivateAndActivateCacheValue() throws Exception {
+ String chName = "myCache";
+
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ IgniteCache<String, String> cacheExp = ig1.getOrCreateCache(chName);
+
+ Configuration<String, String> cfgExp = cacheExp.getConfiguration(CacheConfiguration.class);
+
+ cacheExp.put("key", "value");
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig2.active(false);
+
+ IgniteEx ex1 = (IgniteEx)ig1;
+ IgniteEx ex2 = (IgniteEx)ig2;
+ IgniteEx ex3 = (IgniteEx)ig3;
+
+ GridCacheProcessor cache1 = ex1.context().cache();
+ GridCacheProcessor cache2 = ex2.context().cache();
+ GridCacheProcessor cache3 = ex3.context().cache();
+
+ assertTrue(F.isEmpty(cache1.jcaches()));
+ assertTrue(F.isEmpty(cache2.jcaches()));
+ assertTrue(F.isEmpty(cache3.jcaches()));
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ IgniteCache<String, String> cacheAct = ig2.cache(chName);
+
+ Configuration<String, String> cfgAct = cacheAct.getConfiguration(CacheConfiguration.class);
+
+ assertEquals("value", cacheAct.get("key"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java
new file mode 100644
index 0000000..f13ef97
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+
+/**
+ *
+ */
+public class IgniteChangeGlobalStateDataStreamerTest extends IgniteChangeGlobalStateAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected int backUpNodes() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int backUpClientNodes() {
+ return 0;
+ }
+
+ /**
+ *
+ */
+ public void testDeActivateAndActivateDataStreamer() throws InterruptedException {
+
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ Ignite ig1C = primaryClient(0);
+ Ignite ig2C = primaryClient(1);
+ Ignite ig3C = primaryClient(2);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ assertTrue(ig1C.active());
+ assertTrue(ig2C.active());
+ assertTrue(ig3C.active());
+
+ String cacheName = "myStreamCache";
+
+ ig2C.getOrCreateCache(cacheName);
+
+ try (IgniteDataStreamer<Integer, String> stmr = ig1.dataStreamer(cacheName)) {
+ for (int i = 0; i < 100; i++)
+ stmr.addData(i, Integer.toString(i));
+ }
+
+ ig2C.active(false);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ boolean fail = false;
+
+ try {
+ IgniteDataStreamer<String, String> strm2 = ig2.dataStreamer(cacheName);
+ }
+ catch (Exception e) {
+ fail = true;
+
+ assertTrue(e.getMessage().contains("can not perform operation, because cluster inactive"));
+ }
+
+ if (!fail)
+ fail("exception was not throw");
+
+ ig3C.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ assertTrue(ig1C.active());
+ assertTrue(ig2C.active());
+ assertTrue(ig3C.active());
+
+ try (IgniteDataStreamer<Integer, String> stmr2 = ig2.dataStreamer(cacheName)) {
+ for (int i = 100; i < 200; i++)
+ stmr2.addData(i, Integer.toString(i));
+ }
+
+ IgniteCache<Integer, String> cache = ig3.cache(cacheName);
+
+ for (int i = 0; i < 200; i++)
+ assertEquals(String.valueOf(i), cache.get(i));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java
new file mode 100644
index 0000000..9013226
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateDataStructureTest.java
@@ -0,0 +1,278 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteAtomicLong;
+import org.apache.ignite.IgniteAtomicSequence;
+import org.apache.ignite.IgniteCountDownLatch;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/**
+ *
+ */
+public class IgniteChangeGlobalStateDataStructureTest extends IgniteChangeGlobalStateAbstractTest {
+ /**
+ *
+ */
+ public void testDeActivateAndActivateAtomicLong() throws Exception{
+ String lName = "myLong";
+
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ IgniteAtomicLong lexp = ig1.atomicLong(lName, 100, true);
+
+ lexp.incrementAndGet();
+ lexp.incrementAndGet();
+ lexp.incrementAndGet();
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig2.active(false);
+
+ IgniteEx ex1 = (IgniteEx)ig1;
+ IgniteEx ex2 = (IgniteEx)ig2;
+ IgniteEx ex3 = (IgniteEx)ig3;
+
+ GridCacheProcessor cache1 = ex1.context().cache();
+ GridCacheProcessor cache2 = ex2.context().cache();
+ GridCacheProcessor cache3 = ex3.context().cache();
+
+ assertTrue(F.isEmpty(cache1.jcaches()));
+ assertTrue(F.isEmpty(cache2.jcaches()));
+ assertTrue(F.isEmpty(cache3.jcaches()));
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ IgniteAtomicLong lact1 = ig1.atomicLong(lName, 0, false);
+ IgniteAtomicLong lact2 = ig2.atomicLong(lName, 0, false);
+ IgniteAtomicLong lact3 = ig3.atomicLong(lName, 0, false);
+
+ assertEquals(103, lact1.get());
+ assertEquals(103, lact2.get());
+ assertEquals(103, lact3.get());
+
+ lact1.incrementAndGet();
+
+ assertEquals(104, lact1.get());
+ assertEquals(104, lact2.get());
+ assertEquals(104, lact3.get());
+
+ lact2.incrementAndGet();
+
+ assertEquals(105, lact1.get());
+ assertEquals(105, lact2.get());
+ assertEquals(105, lact3.get());
+
+ lact3.incrementAndGet();
+
+ assertEquals(106, lact3.get());
+ assertEquals(106, lact3.get());
+ assertEquals(106, lact3.get());
+ }
+
+ /**
+ *
+ */
+ public void testDeActivateAndActivateCountDownLatch() throws Exception {
+ final AtomicInteger cnt = new AtomicInteger();
+
+ String latchName = "myLatch";
+
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ IgniteCountDownLatch latchExp1 = ig1.countDownLatch(latchName, 5, false, true);
+
+ latchExp1.countDown();
+
+ assertEquals(4, latchExp1.count());
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig2.active(false);
+
+ IgniteEx ex1 = (IgniteEx)ig1;
+ IgniteEx ex2 = (IgniteEx)ig2;
+ IgniteEx ex3 = (IgniteEx)ig3;
+
+ GridCacheProcessor cache1 = ex1.context().cache();
+ GridCacheProcessor cache2 = ex2.context().cache();
+ GridCacheProcessor cache3 = ex3.context().cache();
+
+ assertTrue(F.isEmpty(cache1.jcaches()));
+ assertTrue(F.isEmpty(cache2.jcaches()));
+ assertTrue(F.isEmpty(cache3.jcaches()));
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ final IgniteCountDownLatch latchAct1 = ig1.countDownLatch(latchName, 0, false, false);
+ final IgniteCountDownLatch latchAct2 = ig2.countDownLatch(latchName, 0, false, false);
+ final IgniteCountDownLatch latchAct3 = ig3.countDownLatch(latchName, 0, false, false);
+
+ runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ latchAct1.await();
+ cnt.incrementAndGet();
+ return null;
+ }
+ });
+
+ runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ latchAct2.await();
+ cnt.incrementAndGet();
+ return null;
+ }
+ });
+
+ runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ latchAct3.await();
+ cnt.incrementAndGet();
+ return null;
+ }
+ });
+
+
+ assertEquals(4, latchAct1.count());
+ assertEquals(4, latchAct2.count());
+ assertEquals(4, latchAct3.count());
+
+ latchAct1.countDown();
+ latchAct2.countDown();
+ latchAct3.countDown();
+
+ assertEquals(1, latchAct1.count());
+ assertEquals(1, latchAct2.count());
+ assertEquals(1, latchAct3.count());
+
+ latchAct1.countDown();
+
+ U.sleep(3000);
+
+ assertEquals(3, cnt.get());
+ }
+
+ /**
+ *
+ */
+ public void testDeActivateAndActivateAtomicSequence(){
+ String seqName = "mySeq";
+
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ IgniteAtomicSequence seqExp1 = ig1.atomicSequence(seqName, 0, true);
+ IgniteAtomicSequence seqExp2 = ig2.atomicSequence(seqName, 0, false);
+ IgniteAtomicSequence seqExp3 = ig3.atomicSequence(seqName, 0, false);
+
+ assertEquals(0, seqExp1.get());
+ assertEquals(1000, seqExp2.get());
+ assertEquals(2000, seqExp3.get());
+
+ seqExp1.incrementAndGet();
+ seqExp2.incrementAndGet();
+ seqExp3.incrementAndGet();
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig2.active(false);
+
+ IgniteEx ex1 = (IgniteEx)ig1;
+ IgniteEx ex2 = (IgniteEx)ig2;
+ IgniteEx ex3 = (IgniteEx)ig3;
+
+ GridCacheProcessor cache1 = ex1.context().cache();
+ GridCacheProcessor cache2 = ex2.context().cache();
+ GridCacheProcessor cache3 = ex3.context().cache();
+
+ assertTrue(F.isEmpty(cache1.jcaches()));
+ assertTrue(F.isEmpty(cache2.jcaches()));
+ assertTrue(F.isEmpty(cache3.jcaches()));
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ IgniteAtomicSequence seqAct1 = ig1.atomicSequence(seqName, 0, false);
+ IgniteAtomicSequence seqAct2 = ig2.atomicSequence(seqName, 0, false);
+ IgniteAtomicSequence seqAct3 = ig3.atomicSequence(seqName, 0, false);
+
+ assertEquals(1, seqAct1.get());
+ assertEquals(1001, seqAct2.get());
+ assertEquals(2001, seqAct3.get());
+
+ seqAct1.incrementAndGet();
+
+ assertEquals(2, seqAct1.get());
+ assertEquals(1001, seqAct2.get());
+ assertEquals(2001, seqAct3.get());
+
+ seqAct2.incrementAndGet();
+
+ assertEquals(2, seqAct1.get());
+ assertEquals(1002, seqAct2.get());
+ assertEquals(2001, seqAct3.get());
+
+ seqAct3.incrementAndGet();
+
+ assertEquals(2, seqAct1.get());
+ assertEquals(1002, seqAct2.get());
+ assertEquals(2002, seqAct3.get());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
new file mode 100644
index 0000000..bb05048
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateFailOverTest.java
@@ -0,0 +1,353 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.internal.IgniteInternalFuture;
+
+import static java.lang.Thread.sleep;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/**
+ *
+ */
+public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalStateAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected int primaryNodes() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int primaryClientNodes() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int backUpClientNodes() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int backUpNodes() {
+ return 4;
+ }
+
+ /**
+ *
+ */
+ public void testActivateDeActivateOnFixTopology() throws Exception {
+ final Ignite igB1 = backUp(0);
+ final Ignite igB2 = backUp(1);
+ final Ignite igB3 = backUp(2);
+
+ assertTrue(!igB1.active());
+ assertTrue(!igB2.active());
+ assertTrue(!igB3.active());
+
+ final AtomicInteger cntA = new AtomicInteger();
+ final AtomicInteger cntD = new AtomicInteger();
+
+ final AtomicLong timeA = new AtomicLong();
+ final AtomicLong timeD = new AtomicLong();
+
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ final AtomicBoolean canAct = new AtomicBoolean(true);
+
+ try {
+ final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get()) {
+ Ignite ig = randomBackUp(false);
+
+ if (canAct.get()) {
+ long start = System.currentTimeMillis();
+
+ ig.active(true);
+
+ timeA.addAndGet((System.currentTimeMillis() - start));
+
+ cntA.incrementAndGet();
+
+ for (Ignite ign : allBackUpNodes())
+ assertTrue(ign.active());
+
+ canAct.set(false);
+ }
+
+ }
+ return null;
+ }
+ });
+
+ final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get()) {
+ Ignite ig = randomBackUp(false);
+
+ if (!canAct.get()) {
+ long start = System.currentTimeMillis();
+
+ ig.active(false);
+
+ timeD.addAndGet((System.currentTimeMillis() - start));
+
+ cntD.incrementAndGet();
+
+ for (Ignite ign : allBackUpNodes())
+ assertTrue(!ign.active());
+
+ canAct.set(true);
+ }
+
+ }
+ return null;
+ }
+ });
+
+ sleep(60_000);
+
+ stop.set(true);
+
+ af.get();
+ df.get();
+ }
+ finally {
+ log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:"
+ + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()) + " nodes: " + backUpNodes()));
+ }
+ }
+
+ /**
+ *
+ */
+ public void testActivateDeActivateOnJoiningNode() throws Exception {
+ final Ignite igB1 = backUp(0);
+ final Ignite igB2 = backUp(1);
+ final Ignite igB3 = backUp(2);
+
+ assertTrue(!igB1.active());
+ assertTrue(!igB2.active());
+ assertTrue(!igB3.active());
+
+ final AtomicInteger cntA = new AtomicInteger();
+ final AtomicInteger cntD = new AtomicInteger();
+
+ final AtomicLong timeA = new AtomicLong();
+ final AtomicLong timeD = new AtomicLong();
+
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ final AtomicBoolean canAct = new AtomicBoolean(true);
+ final AtomicInteger seqIdx = new AtomicInteger(backUpNodes());
+
+ try {
+ final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get()) {
+ Ignite ig = randomBackUp(false);
+
+ if (canAct.get()) {
+ long start = System.currentTimeMillis();
+
+ ig.active(true);
+
+ timeA.addAndGet((System.currentTimeMillis() - start));
+
+ cntA.incrementAndGet();
+
+ for (Ignite ign : allBackUpNodes())
+ assertTrue(ign.active());
+
+ canAct.set(false);
+ }
+
+ }
+ return null;
+ }
+ });
+
+ final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get()) {
+ Ignite ig = randomBackUp(false);
+
+ if (!canAct.get()) {
+ long start = System.currentTimeMillis();
+
+ ig.active(false);
+
+ timeD.addAndGet((System.currentTimeMillis() - start));
+
+ cntD.incrementAndGet();
+
+ for (Ignite ign : allBackUpNodes())
+ assertTrue(!ign.active());
+
+ canAct.set(true);
+ }
+
+ }
+ return null;
+ }
+ });
+
+ IgniteInternalFuture<Void> jf1 = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get())
+ startBackUp(seqIdx.incrementAndGet());
+
+ return null;
+ }
+ });
+
+ IgniteInternalFuture<Void> jf2 = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get())
+ startBackUp(seqIdx.incrementAndGet());
+
+ return null;
+ }
+ });
+
+ sleep(60_000);
+
+ stop.set(true);
+
+ af.get();
+ df.get();
+ jf1.get();
+ jf2.get();
+ }
+ finally {
+ log.info("total started nodes: " + (seqIdx.get() - backUpNodes()));
+
+ log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime: "
+ + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()))
+ );
+ }
+ }
+
+ /**
+ *
+ */
+ public void testActivateDeActivateOnFixTopologyWithPutValues() throws Exception {
+ final Ignite igB1 = backUp(0);
+ final Ignite igB2 = backUp(1);
+ final Ignite igB3 = backUp(2);
+
+ assertTrue(!igB1.active());
+ assertTrue(!igB2.active());
+ assertTrue(!igB3.active());
+
+ final CacheConfiguration<String, String> ccfg = new CacheConfiguration<>();
+ ccfg.setName("main-cache");
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ final AtomicInteger cntA = new AtomicInteger();
+ final AtomicInteger cntD = new AtomicInteger();
+
+ final AtomicLong timeA = new AtomicLong();
+ final AtomicLong timeD = new AtomicLong();
+
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ final AtomicBoolean canAct = new AtomicBoolean(true);
+
+ try {
+ final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get()) {
+ Ignite ig = randomBackUp(false);
+
+ if (canAct.get()) {
+ long start = System.currentTimeMillis();
+
+ ig.active(true);
+
+ IgniteCache<String, String> cache = ig.getOrCreateCache(ccfg);
+
+ cache.put("key" + cnt.get(), "value" + cnt.get());
+
+ cnt.incrementAndGet();
+
+ timeA.addAndGet((System.currentTimeMillis() - start));
+
+ cntA.incrementAndGet();
+
+ for (Ignite ign : allBackUpNodes())
+ assertTrue(ign.active());
+
+ canAct.set(false);
+ }
+
+ }
+ return null;
+ }
+ });
+
+ final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (!stop.get()) {
+ Ignite ig = randomBackUp(false);
+
+ if (!canAct.get()) {
+ long start = System.currentTimeMillis();
+
+ IgniteCache<String, String> cache = ig.getOrCreateCache(ccfg);
+
+ for (int i = 0; i < cnt.get(); i++)
+ assertEquals("value" + i, cache.get("key" + i));
+
+ ig.active(false);
+
+ timeD.addAndGet((System.currentTimeMillis() - start));
+
+ cntD.incrementAndGet();
+
+ for (Ignite ign : allBackUpNodes())
+ assertTrue(!ign.active());
+
+ canAct.set(true);
+ }
+
+ }
+ return null;
+ }
+ });
+
+ sleep(60_000);
+
+ stop.set(true);
+
+ af.get();
+ df.get();
+ }
+ finally {
+ log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:"
+ + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()) + " nodes: " + backUpNodes()));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateServiceTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateServiceTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateServiceTest.java
new file mode 100644
index 0000000..44e0357
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateServiceTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.Collection;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceConfiguration;
+import org.apache.ignite.services.ServiceContext;
+import org.apache.ignite.services.ServiceDescriptor;
+
+/**
+ *
+ */
+public class IgniteChangeGlobalStateServiceTest extends IgniteChangeGlobalStateAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected int backUpClientNodes() {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int primaryNodes() {
+ return 1;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected int backUpNodes() {
+ return 1;
+ }
+
+ /**
+ *
+ */
+ public void testDeployService() throws Exception {
+ Ignite ig1P = primary(0);
+
+ Ignite ig1B = backUp(0);
+
+ String serName = "service";
+
+ ServiceConfiguration serConf = new ServiceConfiguration();
+ serConf.setTotalCount(1);
+ serConf.setName(serName);
+ serConf.setService(new TestService());
+
+ ig1P.services().deploy(serConf);
+
+ stopAllPrimary();
+
+ ig1B.active(true);
+
+ U.sleep(3000);
+
+ Collection<ServiceDescriptor> descs = ig1B.services().serviceDescriptors();
+
+ assertTrue(!F.isEmpty(descs));
+
+ TestService srv = ig1B.services().service(serName);
+
+ assertTrue(srv != null);
+ }
+
+ /**
+ *
+ */
+ private static class TestService implements Service{
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public void cancel(ServiceContext ctx) {
+ System.out.println("cancel service");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void init(ServiceContext ctx) throws Exception {
+ System.out.println("init service");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void execute(ServiceContext ctx) throws Exception {
+ System.out.println("execute service");
+ }
+ }
+}
[12/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileInput.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileInput.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileInput.java
deleted file mode 100644
index 1ba002f..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileInput.java
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.PureJavaCrc32;
-import org.jetbrains.annotations.NotNull;
-
-/**
- * File input.
- */
-public final class FileInput implements ByteBufferBackedDataInput {
- /** */
- private ByteBuffer buf;
-
- /** */
- private FileChannel ch;
-
- /** */
- private long pos;
-
- /**
- * @param ch Channel.
- * @param buf Buffer.
- */
- public FileInput(FileChannel ch, ByteBuffer buf) throws IOException {
- assert ch != null;
-
- this.ch = ch;
- this.buf = buf;
-
- pos = ch.position();
-
- clearBuffer();
- }
-
- /**
- * Clear buffer.
- */
- private void clearBuffer() {
- buf.clear();
- buf.limit(0);
-
- assert buf.remaining() == 0; // Buffer is empty.
- }
-
- /**
- * @param pos Position in bytes from file begin.
- */
- public void seek(long pos) throws IOException {
- if (pos > ch.size())
- throw new EOFException();
-
- ch.position(pos);
-
- this.pos = pos;
-
- clearBuffer();
- }
-
- /**
- * @return Underlying buffer.
- */
- @Override public ByteBuffer buffer() {
- return buf;
- }
-
-
- /** {@inheritDoc} */
- @Override public void ensure(int requested) throws IOException {
- int available = buf.remaining();
-
- if (available >= requested)
- return;
-
- if (buf.capacity() < requested)
- throw new IOException("Requested size is greater than buffer: " + requested);
-
- buf.compact();
-
- do {
- int read = ch.read(buf);
-
- if (read == -1)
- throw new EOFException();
-
- available += read;
-
- pos += read;
- }
- while (available < requested);
-
- buf.flip();
- }
-
- /**
- * @return Position in the stream.
- */
- public long position() {
- return pos - buf.remaining();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public void readFully(@NotNull byte[] b) throws IOException {
- ensure(b.length);
-
- buf.get(b);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
- ensure(len);
-
- buf.get(b, off, len);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int skipBytes(int n) throws IOException {
- if (buf.remaining() >= n)
- buf.position(buf.position() + n);
- else
- seek(pos + n);
-
- return n;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean readBoolean() throws IOException {
- return readByte() == 1;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public byte readByte() throws IOException {
- ensure(1);
-
- return buf.get();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readUnsignedByte() throws IOException {
- return readByte() & 0xFF;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public short readShort() throws IOException {
- ensure(2);
-
- return buf.getShort();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readUnsignedShort() throws IOException {
- return readShort() & 0xFFFF;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public char readChar() throws IOException {
- ensure(2);
-
- return buf.getChar();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readInt() throws IOException {
- ensure(4);
-
- return buf.getInt();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public long readLong() throws IOException {
- ensure(8);
-
- return buf.getLong();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public float readFloat() throws IOException {
- ensure(4);
-
- return buf.getFloat();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public double readDouble() throws IOException {
- ensure(8);
-
- return buf.getDouble();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String readLine() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String readUTF() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * @param skipCheck If CRC check should be skipped.
- * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one
- */
- public Crc32CheckingFileInput startRead(boolean skipCheck) {
- return new Crc32CheckingFileInput(buf.position(), skipCheck);
- }
-
- /**
- * Checking of CRC32.
- */
- public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable {
- /** */
- private final PureJavaCrc32 crc32 = new PureJavaCrc32();
-
- /** Last calc position. */
- private int lastCalcPosition;
-
- /** Skip crc check. */
- private boolean skipCheck;
-
- /**
- * @param position Position.
- */
- public Crc32CheckingFileInput(int position, boolean skipCheck) {
- this.lastCalcPosition = position;
- this.skipCheck = skipCheck;
- }
-
- /** {@inheritDoc} */
- @Override public void ensure(int requested) throws IOException {
- int available = buf.remaining();
-
- if (available >= requested)
- return;
-
- updateCrc();
-
- FileInput.this.ensure(requested);
-
- lastCalcPosition = 0;
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws Exception {
- updateCrc();
-
- int val = crc32.getValue();
-
- int writtenCrc = this.readInt();
-
- if ((val ^ writtenCrc) != 0 && !skipCheck) {
- // If it last message we will skip it (EOF will be thrown).
- ensure(5);
-
- throw new IgniteDataIntegrityViolationException(
- "val: " + val + " writtenCrc: " + writtenCrc
- );
- }
- }
-
- /**
- *
- */
- private void updateCrc() {
- if (skipCheck)
- return;
-
- int oldPos = buf.position();
-
- buf.position(lastCalcPosition);
-
- crc32.update(buf, oldPos - lastCalcPosition);
-
- lastCalcPosition = oldPos;
- }
-
- /** {@inheritDoc} */
- @Override public int skipBytes(int n) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public void readFully(@NotNull byte[] b) throws IOException {
- ensure(b.length);
-
- buf.get(b);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
- ensure(len);
-
- buf.get(b, off, len);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public boolean readBoolean() throws IOException {
- return readByte() == 1;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public byte readByte() throws IOException {
- ensure(1);
-
- return buf.get();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readUnsignedByte() throws IOException {
- return readByte() & 0xFF;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public short readShort() throws IOException {
- ensure(2);
-
- return buf.getShort();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readUnsignedShort() throws IOException {
- return readShort() & 0xFFFF;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public char readChar() throws IOException {
- ensure(2);
-
- return buf.getChar();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public int readInt() throws IOException {
- ensure(4);
-
- return buf.getInt();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public long readLong() throws IOException {
- ensure(8);
-
- return buf.getLong();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public float readFloat() throws IOException {
- ensure(4);
-
- return buf.getFloat();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public double readDouble() throws IOException {
- ensure(8);
-
- return buf.getDouble();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String readLine() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override public String readUTF() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public ByteBuffer buffer() {
- return FileInput.this.buffer();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWALPointer.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWALPointer.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWALPointer.java
deleted file mode 100644
index 8884fb3..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWALPointer.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.util.typedef.internal.S;
-
-/**
- * File WAL pointer.
- */
-public class FileWALPointer implements WALPointer, Comparable<FileWALPointer> {
- /** */
- private final long idx;
-
- /** */
- private final int fileOffset;
-
- /** Written record length */
- private int len;
-
- /** Force flush flag. Used in BACKGROUND WAL mode. */
- private boolean forceFlush;
-
- /**
- * @param idx File timestamp index.
- * @param fileOffset Offset in file, from the beginning.
- * @param len Record length.
- */
- public FileWALPointer(long idx, int fileOffset, int len) {
- this(idx, fileOffset, len, false);
- }
-
- /**
- * @param idx File timestamp index.
- * @param fileOffset Offset in file, from the beginning.
- * @param len Record length.
- * @param forceFlush Force flush flag.
- */
- public FileWALPointer(long idx, int fileOffset, int len, boolean forceFlush) {
- this.idx = idx;
- this.fileOffset = fileOffset;
- this.len = len;
- this.forceFlush = forceFlush;
- }
-
- /**
- * @return Timestamp index.
- */
- public long index() {
- return idx;
- }
-
- /**
- * @return File offset.
- */
- public int fileOffset() {
- return fileOffset;
- }
-
- /**
- * @return Record length.
- */
- public int length() {
- return len;
- }
-
- /**
- * @param len Record length.
- */
- public void length(int len) {
- this.len = len;
- }
-
- /** {@inheritDoc} */
- @Override public WALPointer next() {
- if (len == 0)
- throw new IllegalStateException("Failed to calculate next WAL pointer " +
- "(this pointer is a terminal): " + this);
-
- // Return a terminal pointer.
- return new FileWALPointer(idx, fileOffset + len, 0);
- }
-
- /**
- * @return Force flush flag.
- */
- public boolean forceFlush() {
- return forceFlush;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof FileWALPointer))
- return false;
-
- FileWALPointer that = (FileWALPointer)o;
-
- return idx == that.idx && fileOffset == that.fileOffset;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int result = (int)(idx ^ (idx >>> 32));
-
- result = 31 * result + fileOffset;
-
- return result;
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(FileWALPointer o) {
- int res = Long.compare(idx, o.idx);
-
- return res == 0 ? Integer.compare(fileOffset, o.fileOffset) : res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(FileWALPointer.class, this);
- }
-}
[33/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
new file mode 100755
index 0000000..423d572
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -0,0 +1,2313 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.OffheapReadWriteLock;
+import org.apache.ignite.internal.util.future.CountDownFuture;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.lang.GridPredicate3;
+import org.apache.ignite.internal.util.offheap.GridOffHeapOutOfMemoryException;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import sun.misc.JavaNioAccess;
+import sun.misc.SharedSecrets;
+import sun.nio.ch.DirectBuffer;
+
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+
+/**
+ * Page header structure is described by the following diagram.
+ *
+ * When page is not allocated (in a free list):
+ * <pre>
+ * +--------+------------------------------------------------------+
+ * |8 bytes | PAGE_SIZE + PAGE_OVERHEAD - 8 bytes |
+ * +--------+------------------------------------------------------+
+ * |Next ptr| Page data |
+ * +--------+------------------------------------------------------+
+ * </pre>
+ * <p/>
+ * When page is allocated and is in use:
+ * <pre>
+ * +------------------+--------+--------+----+----+--------+--------+----------------------+
+ * | 8 bytes |8 bytes |8 bytes |4 b |4 b |8 bytes |8 bytes | PAGE_SIZE |
+ * +------------------+--------+--------+----+----+--------+--------+----------------------+
+ * | Marker/Timestamp |Rel ptr |Page ID |C ID|PIN | LOCK |TMP BUF | Page data |
+ * +------------------+--------+--------+----+----+--------+--------+----------------------+
+ * </pre>
+ *
+ * Note that first 8 bytes of page header are used either for page marker or for next relative pointer depending
+ * on whether the page is in use or not.
+ */
+@SuppressWarnings({"LockAcquiredButNotSafelyReleased", "FieldAccessedSynchronizedAndUnsynchronized"})
+public class PageMemoryImpl implements PageMemoryEx {
+ /** */
+ public static final long PAGE_MARKER = 0x0000000000000001L;
+
+ /** Relative pointer chunk index mask. */
+ private static final long SEGMENT_INDEX_MASK = 0xFFFFFF0000000000L;
+
+ /** Full relative pointer mask. */
+ private static final long RELATIVE_PTR_MASK = 0xFFFFFFFFFFFFFFL;
+
+ /** Dirty flag. */
+ private static final long DIRTY_FLAG = 0x0100000000000000L;
+
+ /** Dirty flag. */
+ private static final long TMP_DIRTY_FLAG = 0x0200000000000000L;
+
+ /** Invalid relative pointer value. */
+ private static final long INVALID_REL_PTR = RELATIVE_PTR_MASK;
+
+ /** */
+ private static final long OUTDATED_REL_PTR = INVALID_REL_PTR + 1;
+
+ /** Address mask to avoid ABA problem. */
+ private static final long ADDRESS_MASK = 0xFFFFFFFFFFFFFFL;
+
+ /** Counter mask to avoid ABA problem. */
+ private static final long COUNTER_MASK = ~ADDRESS_MASK;
+
+ /** Counter increment to avoid ABA problem. */
+ private static final long COUNTER_INC = ADDRESS_MASK + 1;
+
+ /** Page relative pointer. Does not change once a page is allocated. */
+ public static final int RELATIVE_PTR_OFFSET = 8;
+
+ /** Page ID offset */
+ public static final int PAGE_ID_OFFSET = 16;
+
+ /** Page cache ID offset. */
+ public static final int PAGE_CACHE_ID_OFFSET = 24;
+
+ /** Page pin counter offset. */
+ public static final int PAGE_PIN_CNT_OFFSET = 28;
+
+ /** Page lock offset. */
+ public static final int PAGE_LOCK_OFFSET = 32;
+
+ /** Page lock offset. */
+ public static final int PAGE_TMP_BUF_OFFSET = 40;
+
+ /**
+ * 8b Marker/timestamp
+ * 8b Relative pointer
+ * 8b Page ID
+ * 4b Cache ID
+ * 4b Pin count
+ * 8b Lock
+ * 8b Temporary buffer
+ */
+ public static final int PAGE_OVERHEAD = 48;
+
+ /** Number of random pages that will be picked for eviction. */
+ public static final int RANDOM_PAGES_EVICT_NUM = 5;
+
+ /** Tracking io. */
+ private static final TrackingPageIO trackingIO = TrackingPageIO.VERSIONS.latest();
+
+ /** Page size. */
+ private final int sysPageSize;
+
+ /** Shared context. */
+ private final GridCacheSharedContext<?, ?> sharedCtx;
+
+ /** State checker. */
+ private final CheckpointLockStateChecker stateChecker;
+
+ /** */
+ private Executor asyncRunner = new ThreadPoolExecutor(
+ 0,
+ Runtime.getRuntime().availableProcessors(),
+ 30L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<Runnable>(Runtime.getRuntime().availableProcessors()));
+
+ /** Page store manager. */
+ private IgnitePageStoreManager storeMgr;
+
+ /** */
+ private IgniteWriteAheadLogManager walMgr;
+
+ /** Direct byte buffer factory. */
+ private JavaNioAccess nioAccess;
+
+ /** */
+ private final IgniteLogger log;
+
+ /** Direct memory allocator. */
+ private final DirectMemoryProvider directMemoryProvider;
+
+ /** Segments array. */
+ private Segment[] segments;
+
+ /** */
+ private PagePool checkpointPool;
+
+ /** */
+ private OffheapReadWriteLock rwLock;
+
+ /** Flush dirty page closure. When possible, will be called by evictPage(). */
+ private final GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage;
+
+ /** Flush dirty page closure. When possible, will be called by evictPage(). */
+ private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
+
+ /** */
+ private boolean pageEvictWarned;
+
+ /** */
+ private long[] sizes;
+
+ /** */
+ private MemoryMetricsImpl memMetrics;
+
+ /**
+ * @param directMemoryProvider Memory allocator to use.
+ * @param sharedCtx Cache shared context.
+ * @param pageSize Page size.
+ * @param flushDirtyPage Callback invoked when a dirty page is evicted.
+ * @param changeTracker Callback invoked to track changes in pages.
+ */
+ public PageMemoryImpl(
+ DirectMemoryProvider directMemoryProvider,
+ long[] sizes,
+ GridCacheSharedContext<?, ?> sharedCtx,
+ int pageSize,
+ GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage,
+ GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
+ CheckpointLockStateChecker stateChecker,
+ MemoryMetricsImpl memMetrics
+ ) {
+ assert sharedCtx != null;
+
+ log = sharedCtx.logger(PageMemoryImpl.class);
+
+ this.sharedCtx = sharedCtx;
+ this.directMemoryProvider = directMemoryProvider;
+ this.sizes = sizes;
+ this.flushDirtyPage = flushDirtyPage;
+ this.changeTracker = changeTracker;
+ this.stateChecker = stateChecker;
+
+ storeMgr = sharedCtx.pageStore();
+ walMgr = sharedCtx.wal();
+
+ assert storeMgr != null;
+ assert walMgr != null;
+
+ sysPageSize = pageSize + PAGE_OVERHEAD;
+
+ rwLock = new OffheapReadWriteLock(128);
+
+ this.memMetrics = memMetrics;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ directMemoryProvider.initialize(sizes);
+
+ List<DirectMemoryRegion> regions = new ArrayList<>(sizes.length);
+
+ while (true) {
+ DirectMemoryRegion reg = directMemoryProvider.nextRegion();
+
+ if (reg == null)
+ break;
+
+ regions.add(reg);
+ }
+
+ nioAccess = SharedSecrets.getJavaNioAccess();
+
+ int regs = regions.size();
+
+ segments = new Segment[regs - 1];
+
+ DirectMemoryRegion cpReg = regions.get(regs - 1);
+
+ checkpointPool = new PagePool(regs - 1, cpReg);
+
+ long checkpointBuf = cpReg.size();
+
+ long totalAllocated = 0;
+ int pages = 0;
+ long totalTblSize = 0;
+
+ for (int i = 0; i < regs - 1; i++) {
+ assert i < segments.length;
+
+ DirectMemoryRegion reg = regions.get(i);
+
+ totalAllocated += reg.size();
+
+ segments[i] = new Segment(i, regions.get(i), checkpointPool.pages() / segments.length);
+
+ pages += segments[i].pages();
+ totalTblSize += segments[i].tableSize();
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Started page memory [memoryAllocated=" + U.readableSize(totalAllocated, false) +
+ ", pages=" + pages +
+ ", tableSize=" + U.readableSize(totalTblSize, false) +
+ ", checkpointBuffer=" + U.readableSize(checkpointBuf, false) +
+ ']');
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("OverlyStrongTypeCast")
+ @Override public void stop() throws IgniteException {
+ if (log.isDebugEnabled())
+ log.debug("Stopping page memory.");
+
+ directMemoryProvider.shutdown();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void releasePage(int cacheId, long pageId, long page) {
+ Segment seg = segment(cacheId, pageId);
+
+ seg.readLock().lock();
+
+ try {
+ seg.releasePage(page);
+ }
+ finally {
+ seg.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long readLock(int cacheId, long pageId, long page) {
+ return readLockPage(page, new FullPageId(pageId, cacheId), false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readUnlock(int cacheId, long pageId, long page) {
+ readUnlockPage(page);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long writeLock(int cacheId, long pageId, long page) {
+ return writeLock(cacheId, pageId, page, false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long writeLock(int cacheId, long pageId, long page, boolean restore) {
+ return writeLockPage(page, new FullPageId(pageId, cacheId), !restore);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long tryWriteLock(int cacheId, long pageId, long page) {
+ return tryWriteLockPage(page, new FullPageId(pageId, cacheId), true);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+ boolean dirtyFlag) {
+ writeUnlock(cacheId, pageId, page, walPlc, dirtyFlag, false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
+ boolean dirtyFlag, boolean restore) {
+ writeUnlockPage(page, new FullPageId(pageId, cacheId), walPlc, dirtyFlag, restore);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isDirty(int cacheId, long pageId, long page) {
+ return isDirty(page);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
+ assert flags == PageIdAllocator.FLAG_DATA && partId <= PageIdAllocator.MAX_PARTITION_ID ||
+ flags == PageIdAllocator.FLAG_IDX && partId == PageIdAllocator.INDEX_PARTITION :
+ "flags = " + flags + ", partId = " + partId;
+
+ long pageId = storeMgr.allocatePage(cacheId, partId, flags);
+
+ assert PageIdUtils.pageIndex(pageId) > 0; //it's crucial for tracking pages (zero page is super one)
+
+ // We need to allocate page in memory for marking it dirty to save it in the next checkpoint.
+ // Otherwise it is possible that on file will be empty page which will be saved at snapshot and read with error
+ // because there is no crc inside them.
+ Segment seg = segment(cacheId, pageId);
+
+ FullPageId fullId = new FullPageId(pageId, cacheId);
+
+ seg.writeLock().lock();
+
+ boolean isTrackingPage = trackingIO.trackingPageFor(pageId, pageSize()) == pageId;
+
+ try {
+ long relPtr = seg.borrowOrAllocateFreePage(pageId);
+
+ if (relPtr == INVALID_REL_PTR)
+ relPtr = seg.evictPage();
+
+ long absPtr = seg.absolute(relPtr);
+
+ GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
+
+ PageHeader.fullPageId(absPtr, fullId);
+ PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
+ rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
+
+ assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+
+ assert !PageHeader.isAcquired(absPtr) :
+ "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) +
+ ", absPtr=" + U.hexLong(absPtr) + ']';
+
+ setDirty(fullId, absPtr, true, true);
+
+ if (isTrackingPage) {
+ long pageAddr = absPtr + PAGE_OVERHEAD;
+
+ // We are inside segment write lock, so no other thread can pin this tracking page yet.
+ // We can modify page buffer directly.
+ if (PageIO.getType(pageAddr) == 0) {
+ trackingIO.initNewPage(pageAddr, pageId, pageSize());
+
+ if (!sharedCtx.wal().isAlwaysWriteFullPages())
+ sharedCtx.wal().log(
+ new InitNewPageRecord(
+ cacheId,
+ pageId,
+ trackingIO.getType(),
+ trackingIO.getVersion(), pageId
+ )
+ );
+ else
+ sharedCtx.wal().log(new PageSnapshot(fullId, absPtr + PAGE_OVERHEAD, pageSize()));
+ }
+ }
+
+ seg.loadedPages.put(cacheId, PageIdUtils.effectivePageId(pageId), relPtr, seg.partTag(cacheId, partId));
+ }
+ finally {
+ seg.writeLock().unlock();
+ }
+
+ //we have allocated 'tracking' page, we need to allocate regular one
+ return isTrackingPage ? allocatePage(cacheId, partId, flags) : pageId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ByteBuffer pageBuffer(long pageAddr) {
+ return wrapPointer(pageAddr, pageSize());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean freePage(int cacheId, long pageId) throws IgniteCheckedException {
+ assert false : "Free page should be never called directly when persistence is enabled.";
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long metaPageId(int cacheId) throws IgniteCheckedException {
+ return storeMgr.metaPageId(cacheId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException {
+ return PageIdUtils.pageId(partId, PageIdAllocator.FLAG_DATA, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long acquirePage(int cacheId, long pageId) throws IgniteCheckedException {
+ return acquirePage(cacheId, pageId, false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException {
+ FullPageId fullId = new FullPageId(pageId, cacheId);
+
+ int partId = PageIdUtils.partId(pageId);
+
+ Segment seg = segment(cacheId, pageId);
+
+ seg.readLock().lock();
+
+ try {
+ long relPtr = seg.loadedPages.get(
+ cacheId,
+ PageIdUtils.effectivePageId(pageId),
+ seg.partTag(cacheId, partId),
+ INVALID_REL_PTR,
+ INVALID_REL_PTR
+ );
+
+ // The page is loaded to the memory.
+ if (relPtr != INVALID_REL_PTR) {
+ long absPtr = seg.absolute(relPtr);
+
+ seg.acquirePage(absPtr);
+
+ return absPtr;
+ }
+ }
+ finally {
+ seg.readLock().unlock();
+ }
+
+ seg.writeLock().lock();
+
+ try {
+ // Double-check.
+ long relPtr = seg.loadedPages.get(
+ cacheId,
+ PageIdUtils.effectivePageId(pageId),
+ seg.partTag(cacheId, partId),
+ INVALID_REL_PTR,
+ OUTDATED_REL_PTR
+ );
+
+ long absPtr;
+
+ if (relPtr == INVALID_REL_PTR) {
+ relPtr = seg.borrowOrAllocateFreePage(pageId);
+
+ if (relPtr == INVALID_REL_PTR)
+ relPtr = seg.evictPage();
+
+ absPtr = seg.absolute(relPtr);
+
+ PageHeader.fullPageId(absPtr, fullId);
+ PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
+
+ assert !PageHeader.isAcquired(absPtr) :
+ "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) +
+ ", absPtr=" + U.hexLong(absPtr) + ']';
+
+ // We can clear dirty flag after the page has been allocated.
+ setDirty(fullId, absPtr, false, false);
+
+ seg.loadedPages.put(
+ cacheId,
+ PageIdUtils.effectivePageId(pageId),
+ relPtr,
+ seg.partTag(cacheId, partId)
+ );
+
+ long pageAddr = absPtr + PAGE_OVERHEAD;
+
+ if (!restore) {
+ try {
+ ByteBuffer buf = wrapPointer(pageAddr, pageSize());
+
+ memMetrics.updatePageReplaceRate();
+
+ storeMgr.read(cacheId, pageId, buf);
+ }
+ catch (IgniteDataIntegrityViolationException ignore) {
+ U.warn(log, "Failed to read page (data integrity violation encountered, will try to " +
+ "restore using existing WAL) [fullPageId=" + fullId + ']');
+
+ tryToRestorePage(fullId, absPtr);
+
+ seg.acquirePage(absPtr);
+
+ return absPtr;
+ }
+ }
+ else {
+ GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
+
+ // Must init page ID in order to ensure RWLock tag consistency.
+ PageIO.setPageId(pageAddr, pageId);
+ }
+
+ rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
+ }
+ else if (relPtr == OUTDATED_REL_PTR) {
+ assert PageIdUtils.pageIndex(pageId) == 0 : fullId;
+
+ relPtr = refreshOutdatedPage(seg, cacheId, pageId, false);
+
+ absPtr = seg.absolute(relPtr);
+
+ long pageAddr = absPtr + PAGE_OVERHEAD;
+
+ GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
+
+ PageHeader.fullPageId(absPtr, fullId);
+ PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
+ PageIO.setPageId(pageAddr, pageId);
+
+ assert !PageHeader.isAcquired(absPtr) :
+ "Pin counter must be 0 for a new page [relPtr=" + U.hexLong(relPtr) +
+ ", absPtr=" + U.hexLong(absPtr) + ']';
+
+ rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
+ }
+ else
+ absPtr = seg.absolute(relPtr);
+
+ seg.acquirePage(absPtr);
+
+ return absPtr;
+ }
+ finally {
+ seg.writeLock().unlock();
+ }
+ }
+
+ /**
+ * @param seg Segment.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param rmv {@code True} if page should be removed.
+ * @return Relative pointer to refreshed page.
+ */
+ private long refreshOutdatedPage(Segment seg, int cacheId, long pageId, boolean rmv) {
+ assert seg.writeLock().isHeldByCurrentThread();
+
+ int tag = seg.partTag(cacheId, PageIdUtils.partId(pageId));
+
+ long relPtr = seg.loadedPages.refresh(cacheId, PageIdUtils.effectivePageId(pageId), tag);
+
+ long absPtr = seg.absolute(relPtr);
+
+ GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
+
+ long tmpBufPtr = PageHeader.tempBufferPointer(absPtr);
+
+ if (tmpBufPtr != INVALID_REL_PTR) {
+ GridUnsafe.setMemory(checkpointPool.absolute(tmpBufPtr) + PAGE_OVERHEAD, pageSize(), (byte)0);
+
+ PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
+ PageHeader.dirty(absPtr, false);
+ PageHeader.tempDirty(absPtr, false);
+
+ // We pinned the page when allocated the temp buffer, release it now.
+ PageHeader.releasePage(absPtr);
+
+ checkpointPool.releaseFreePage(tmpBufPtr);
+ }
+
+ if (rmv)
+ seg.loadedPages.remove(cacheId, PageIdUtils.effectivePageId(pageId), tag);
+
+ return relPtr;
+ }
+
+ /**
+ * @param fullId Full id.
+ * @param absPtr Absolute pointer.
+ */
+ private void tryToRestorePage(FullPageId fullId, long absPtr) throws IgniteCheckedException {
+ Long tmpAddr = null;
+
+ try {
+ ByteBuffer curPage = null;
+ ByteBuffer lastValidPage = null;
+
+ for (IgniteBiTuple<WALPointer, WALRecord> tuple : walMgr.replay(null)) {
+ switch (tuple.getValue().type()) {
+ case PAGE_RECORD:
+ PageSnapshot snapshot = (PageSnapshot)tuple.getValue();
+
+ if (snapshot.fullPageId().equals(fullId)) {
+ if (tmpAddr == null) {
+ assert snapshot.pageData().length <= pageSize() : snapshot.pageData().length;
+
+ tmpAddr = GridUnsafe.allocateMemory(pageSize());
+ }
+
+ if (curPage == null)
+ curPage = wrapPointer(tmpAddr, pageSize());
+
+ PageUtils.putBytes(tmpAddr, 0, snapshot.pageData());
+ }
+
+ break;
+
+ case CHECKPOINT_RECORD:
+ CheckpointRecord rec = (CheckpointRecord)tuple.getValue();
+
+ assert !rec.end();
+
+ if (curPage != null) {
+ lastValidPage = curPage;
+ curPage = null;
+ }
+
+ break;
+
+ case MEMORY_RECOVERY: // It means that previous checkpoint was broken.
+ curPage = null;
+
+ break;
+
+ default:
+ if (tuple.getValue() instanceof PageDeltaRecord) {
+ PageDeltaRecord deltaRecord = (PageDeltaRecord)tuple.getValue();
+
+ if (curPage != null
+ && deltaRecord.pageId() == fullId.pageId()
+ && deltaRecord.cacheId() == fullId.cacheId()) {
+ assert tmpAddr != null;
+
+ deltaRecord.applyDelta(this, tmpAddr);
+ }
+ }
+ }
+ }
+
+ ByteBuffer restored = curPage == null ? lastValidPage : curPage;
+
+ if (restored == null)
+ throw new AssertionError(String.format(
+ "Page is broken. Can't restore it from WAL. (cacheId = %d, pageId = %X).",
+ fullId.cacheId(), fullId.pageId()
+ ));
+
+ long pageAddr = writeLockPage(absPtr, fullId, false);
+
+ try {
+ pageBuffer(pageAddr).put(restored);
+ }
+ finally {
+ writeUnlockPage(absPtr, fullId, null, true, false);
+ }
+ }
+ finally {
+ if (tmpAddr != null)
+ GridUnsafe.freeMemory(tmpAddr);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int pageSize() {
+ return sysPageSize - PAGE_OVERHEAD;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int systemPageSize() {
+ return sysPageSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean safeToUpdate() {
+ for (Segment segment : segments)
+ if (!segment.safeToUpdate())
+ return false;
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException {
+ Collection[] collections = new Collection[segments.length];
+
+ for (int i = 0; i < segments.length; i++) {
+ Segment seg = segments[i];
+
+ if (seg.segCheckpointPages != null)
+ throw new IgniteException("Failed to begin checkpoint (it is already in progress).");
+
+ collections[i] = seg.segCheckpointPages = seg.dirtyPages;
+
+ seg.dirtyPages = new GridConcurrentHashSet<>();
+ }
+
+ memMetrics.resetDirtyPages();
+
+ return new GridMultiCollectionWrapper<>(collections);
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings({"unchecked", "TooBroadScope"})
+ @Override public void finishCheckpoint() {
+ for (Segment seg : segments)
+ seg.segCheckpointPages = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public Integer getForCheckpoint(FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) {
+ assert tmpBuf.remaining() == pageSize();
+
+ Segment seg = segment(fullId.cacheId(), fullId.pageId());
+
+ long absPtr = 0;
+
+ long relPtr;
+
+ int tag;
+
+ seg.readLock().lock();
+
+ try {
+ tag = seg.partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId()));
+
+ relPtr = seg.loadedPages.get(
+ fullId.cacheId(),
+ PageIdUtils.effectivePageId(fullId.pageId()),
+ tag,
+ INVALID_REL_PTR,
+ OUTDATED_REL_PTR
+ );
+
+ // Page may have been cleared during eviction. We have nothing to do in this case.
+ if (relPtr == INVALID_REL_PTR)
+ return null;
+
+ if (relPtr != OUTDATED_REL_PTR) {
+ absPtr = seg.absolute(relPtr);
+
+ // Pin the page until page will not be copied.
+ if (PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR)
+ PageHeader.acquirePage(absPtr);
+ }
+ }
+ finally {
+ seg.readLock().unlock();
+ }
+
+ if (relPtr == OUTDATED_REL_PTR) {
+ seg.writeLock().lock();
+
+ try {
+ // Double-check.
+ relPtr = seg.loadedPages.get(
+ fullId.cacheId(),
+ PageIdUtils.effectivePageId(fullId.pageId()),
+ seg.partTag(
+ fullId.cacheId(),
+ PageIdUtils.partId(fullId.pageId())
+ ),
+ INVALID_REL_PTR,
+ OUTDATED_REL_PTR
+ );
+
+ if (relPtr == INVALID_REL_PTR)
+ return null;
+
+ if (relPtr == OUTDATED_REL_PTR) {
+ relPtr = refreshOutdatedPage(
+ seg,
+ fullId.cacheId(),
+ PageIdUtils.effectivePageId(fullId.pageId()),
+ true
+ );
+
+ seg.pool.releaseFreePage(relPtr);
+ }
+
+ return null;
+ }
+ finally {
+ seg.writeLock().unlock();
+ }
+
+ }
+ else {
+ copyPageForCheckpoint(absPtr, fullId, tmpBuf, tracker);
+
+ return tag;
+ }
+ }
+
+ /**
+ * @param absPtr Absolute ptr.
+ * @param fullId Full id.
+ * @param tmpBuf Tmp buffer.
+ */
+ private void copyPageForCheckpoint(long absPtr, FullPageId fullId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker) {
+ assert absPtr != 0;
+
+ long tmpRelPtr;
+
+ rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+
+ try {
+ tmpRelPtr = PageHeader.tempBufferPointer(absPtr);
+
+ clearCheckpoint(fullId);
+
+ if (tmpRelPtr != INVALID_REL_PTR)
+ PageHeader.tempBufferPointer(absPtr, INVALID_REL_PTR);
+ else {
+ copyInBuffer(absPtr, tmpBuf);
+
+ PageHeader.dirty(absPtr, false);
+
+ PageHeader.releasePage(absPtr);
+
+ return;
+ }
+ }
+ finally {
+ rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+ }
+
+ assert tmpRelPtr != 0;
+
+ long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
+
+ copyInBuffer(tmpAbsPtr, tmpBuf);
+
+ GridUnsafe.setMemory(tmpAbsPtr + PAGE_OVERHEAD, pageSize(), (byte)0);
+
+ PageHeader.dirty(tmpAbsPtr, false);
+
+ if (tracker != null)
+ tracker.onCowPageWritten();
+
+ checkpointPool.releaseFreePage(tmpRelPtr);
+
+ // We pinned the page when allocated the temp buffer, release it now.
+ PageHeader.releasePage(absPtr);
+ }
+
+ /**
+ * @param absPtr Absolute ptr.
+ * @param tmpBuf Tmp buffer.
+ */
+ private void copyInBuffer(long absPtr, ByteBuffer tmpBuf) {
+ if (tmpBuf.isDirect()) {
+ long tmpPtr = ((DirectBuffer)tmpBuf).address();
+
+ GridUnsafe.copyMemory(absPtr + PAGE_OVERHEAD, tmpPtr, pageSize());
+
+ assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+ assert GridUnsafe.getInt(tmpPtr + 4) == 0; //TODO GG-11480
+ }
+ else {
+ byte[] arr = tmpBuf.array();
+
+ assert arr != null;
+ assert arr.length == pageSize();
+
+ GridUnsafe.copyMemory(null, absPtr + PAGE_OVERHEAD, arr, GridUnsafe.BYTE_ARR_OFF, pageSize());
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int invalidate(int cacheId, int partId) {
+ int tag = 0;
+
+ for (Segment seg : segments) {
+ seg.writeLock().lock();
+
+ try {
+ int newTag = seg.incrementPartTag(cacheId, partId);
+
+ if (tag == 0)
+ tag = newTag;
+
+ assert tag == newTag;
+ }
+ finally {
+ seg.writeLock().unlock();
+ }
+ }
+
+ return tag;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCacheGroupDestroyed(int grpId) {
+ for (Segment seg : segments) {
+ seg.writeLock().lock();
+
+ try {
+ seg.resetPartTags(grpId);
+ }
+ finally {
+ seg.writeLock().unlock();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<Void> clearAsync(
+ GridPredicate3<Integer, Long, Integer> pred,
+ boolean cleanDirty
+ ) {
+ CountDownFuture completeFut = new CountDownFuture(segments.length);
+
+ for (Segment seg : segments) {
+ Runnable clear = new ClearSegmentRunnable(seg, pred, cleanDirty, completeFut, pageSize());
+
+ try {
+ asyncRunner.execute(clear);
+ }
+ catch (RejectedExecutionException ignore) {
+ clear.run();
+ }
+ }
+
+ return completeFut;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long loadedPages() {
+ long total = 0;
+
+ for (Segment seg : segments) {
+ seg.readLock().lock();
+
+ try {
+ total += seg.loadedPages.size();
+ }
+ finally {
+ seg.readLock().unlock();
+ }
+ }
+
+ return total;
+ }
+
+ /**
+ * @return Total number of acquired pages.
+ */
+ public long acquiredPages() {
+ long total = 0;
+
+ for (Segment seg : segments) {
+ seg.readLock().lock();
+
+ try {
+ total += seg.acquiredPages();
+ }
+ finally {
+ seg.readLock().unlock();
+ }
+ }
+
+ return total;
+ }
+
+ /**
+ * @param absPtr Absolute pointer to read lock.
+ * @param force Force flag.
+ * @return Pointer to the page read buffer.
+ */
+ private long readLockPage(long absPtr, FullPageId fullId, boolean force) {
+ int tag = force ? -1 : PageIdUtils.tag(fullId.pageId());
+
+ boolean locked = rwLock.readLock(absPtr + PAGE_LOCK_OFFSET, tag);
+
+ if (!locked)
+ return 0;
+
+ PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
+
+ assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+
+ return absPtr + PAGE_OVERHEAD;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long readLockForce(int cacheId, long pageId, long page) {
+ return readLockPage(page, new FullPageId(pageId, cacheId), true);
+ }
+
+ /**
+ * @param absPtr Absolute pointer to unlock.
+ */
+ void readUnlockPage(long absPtr) {
+ rwLock.readUnlock(absPtr + PAGE_LOCK_OFFSET);
+ }
+
+ /**
+ * Checks if a page has temp copy buffer.
+ *
+ * @param absPtr Absolute pointer.
+ * @return {@code True} if a page has temp buffer.
+ */
+ public boolean hasTempCopy(long absPtr) {
+ return PageHeader.tempBufferPointer(absPtr) != INVALID_REL_PTR;
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @return Pointer to the page write buffer or {@code 0} if page was not locked.
+ */
+ long tryWriteLockPage(long absPtr, FullPageId fullId, boolean checkTag) {
+ int tag = checkTag ? PageIdUtils.tag(fullId.pageId()) : OffheapReadWriteLock.TAG_LOCK_ALWAYS;
+
+ if (!rwLock.tryWriteLock(absPtr + PAGE_LOCK_OFFSET, tag))
+ return 0;
+
+ return postWriteLockPage(absPtr, fullId);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @return Pointer to the page write buffer.
+ */
+ private long writeLockPage(long absPtr, FullPageId fullId, boolean checkTag) {
+ int tag = checkTag ? PageIdUtils.tag(fullId.pageId()) : OffheapReadWriteLock.TAG_LOCK_ALWAYS;
+
+ boolean locked = rwLock.writeLock(absPtr + PAGE_LOCK_OFFSET, tag);
+
+ return locked ? postWriteLockPage(absPtr, fullId) : 0;
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @return Pointer to the page write buffer.
+ */
+ private long postWriteLockPage(long absPtr, FullPageId fullId) {
+ PageHeader.writeTimestamp(absPtr, U.currentTimeMillis());
+
+ // Create a buffer copy if the page is scheduled for a checkpoint.
+ if (isInCheckpoint(fullId) && PageHeader.tempBufferPointer(absPtr) == INVALID_REL_PTR) {
+ long tmpRelPtr = checkpointPool.borrowOrAllocateFreePage(fullId.pageId());
+
+ if (tmpRelPtr == INVALID_REL_PTR) {
+ rwLock.writeUnlock(absPtr + PAGE_LOCK_OFFSET, OffheapReadWriteLock.TAG_LOCK_ALWAYS);
+
+ throw new IgniteException(
+ "Failed to allocate temporary buffer for checkpoint " +
+ "(increase checkpointPageBufferSize configuration property)");
+ }
+
+ // Pin the page until checkpoint is not finished.
+ PageHeader.acquirePage(absPtr);
+
+ long tmpAbsPtr = checkpointPool.absolute(tmpRelPtr);
+
+ assert !PageHeader.tempDirty(tmpAbsPtr);
+
+ GridUnsafe.copyMemory(
+ null,
+ absPtr + PAGE_OVERHEAD,
+ null,
+ tmpAbsPtr + PAGE_OVERHEAD,
+ pageSize()
+ );
+
+ PageHeader.dirty(absPtr, false);
+ PageHeader.tempBufferPointer(absPtr, tmpRelPtr);
+
+ assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+ assert GridUnsafe.getInt(tmpAbsPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+ }
+
+ assert GridUnsafe.getInt(absPtr + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+
+ return absPtr + PAGE_OVERHEAD;
+ }
+
+ /**
+ * @param page Page pointer.
+ * @param walPlc Full page WAL record policy.
+ */
+ private void writeUnlockPage(
+ long page,
+ FullPageId fullId,
+ Boolean walPlc,
+ boolean markDirty,
+ boolean restore
+ ) {
+ boolean dirty = isDirty(page);
+
+ //if page is for restore, we shouldn't mark it as changed
+ if (!restore && markDirty && !dirty)
+ changeTracker.apply(page, fullId, this);
+
+ boolean pageWalRec = markDirty && walPlc != FALSE && (walPlc == TRUE || !dirty);
+
+ assert GridUnsafe.getInt(page + PAGE_OVERHEAD + 4) == 0; //TODO GG-11480
+
+ if (markDirty)
+ setDirty(fullId, page, markDirty, false);
+
+ beforeReleaseWrite(fullId, page + PAGE_OVERHEAD, pageWalRec);
+
+ long pageId = PageIO.getPageId(page + PAGE_OVERHEAD);
+
+ try {
+ rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
+ }
+ catch (AssertionError ex) {
+ StringBuilder sb = new StringBuilder(sysPageSize * 2);
+
+ for (int i = 0; i < systemPageSize(); i += 8)
+ sb.append(U.hexLong(GridUnsafe.getLong(page + i)));
+
+ U.error(log, "Failed to unlock page [fullPageId=" + fullId + ", binPage=" + sb + ']');
+
+ throw ex;
+ }
+ }
+
+ /**
+ * @param absPtr Absolute pointer to the page.
+ * @return {@code True} if write lock acquired for the page.
+ */
+ boolean isPageWriteLocked(long absPtr) {
+ return rwLock.isWriteLocked(absPtr + PAGE_LOCK_OFFSET);
+ }
+
+ /**
+ * @param absPtr Absolute pointer to the page.
+ * @return {@code True} if read lock acquired for the page.
+ */
+ boolean isPageReadLocked(long absPtr) {
+ return rwLock.isReadLocked(absPtr + PAGE_LOCK_OFFSET);
+ }
+
+ /**
+ * @param ptr Pointer to wrap.
+ * @param len Memory location length.
+ * @return Wrapped buffer.
+ */
+ ByteBuffer wrapPointer(long ptr, int len) {
+ ByteBuffer buf = nioAccess.newDirectByteBuffer(ptr, len, null);
+
+ buf.order(ByteOrder.nativeOrder());
+
+ return buf;
+ }
+
+ /**
+ * @param pageId Page ID to check if it was added to the checkpoint list.
+ * @return {@code True} if it was added to the checkpoint list.
+ */
+ boolean isInCheckpoint(FullPageId pageId) {
+ Segment seg = segment(pageId.cacheId(), pageId.pageId());
+
+ Collection<FullPageId> pages0 = seg.segCheckpointPages;
+
+ return pages0 != null && pages0.contains(pageId);
+ }
+
+ /**
+ * @param fullPageId Page ID to clear.
+ */
+ void clearCheckpoint(FullPageId fullPageId) {
+ Segment seg = segment(fullPageId.cacheId(), fullPageId.pageId());
+
+ Collection<FullPageId> pages0 = seg.segCheckpointPages;
+
+ assert pages0 != null;
+
+ pages0.remove(fullPageId);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @return {@code True} if page is dirty.
+ */
+ boolean isDirty(long absPtr) {
+ return PageHeader.dirty(absPtr);
+ }
+
+ /**
+ * Gets the number of active pages across all segments. Used for test purposes only.
+ *
+ * @return Number of active pages.
+ */
+ public int activePagesCount() {
+ int total = 0;
+
+ for (Segment seg : segments)
+ total += seg.acquiredPages();
+
+ return total;
+ }
+
+ /**
+ * This method must be called in synchronized context.
+ *
+ * @param absPtr Absolute pointer.
+ * @param dirty {@code True} dirty flag.
+ * @param forceAdd If this flag is {@code true}, then the page will be added to the dirty set regardless whether the
+ * old flag was dirty or not.
+ */
+ void setDirty(FullPageId pageId, long absPtr, boolean dirty, boolean forceAdd) {
+ boolean wasDirty = PageHeader.dirty(absPtr, dirty);
+
+ if (dirty) {
+ if (!wasDirty || forceAdd) {
+ boolean added = segment(pageId.cacheId(), pageId.pageId()).dirtyPages.add(pageId);
+
+ if (added)
+ memMetrics.incrementDirtyPages();
+ }
+ }
+ else {
+ boolean rmv = segment(pageId.cacheId(), pageId.pageId()).dirtyPages.remove(pageId);
+
+ if (rmv)
+ memMetrics.decrementDirtyPages();
+ }
+ }
+
+ /**
+ *
+ */
+ void beforeReleaseWrite(FullPageId pageId, long ptr, boolean pageWalRec) {
+ if (walMgr != null && (pageWalRec || walMgr.isAlwaysWriteFullPages())) {
+ try {
+ walMgr.log(new PageSnapshot(pageId, ptr, pageSize()));
+ }
+ catch (IgniteCheckedException e) {
+ // TODO ignite-db.
+ throw new IgniteException(e);
+ }
+ }
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @return Segment.
+ */
+ private Segment segment(int cacheId, long pageId) {
+ int idx = segmentIndex(cacheId, pageId, segments.length);
+
+ return segments[idx];
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @return Segment index.
+ */
+ public static int segmentIndex(int cacheId, long pageId, int segments) {
+ pageId = PageIdUtils.effectivePageId(pageId);
+
+ // Take a prime number larger than total number of partitions.
+ int hash = U.hash(pageId * 65537 + cacheId);
+
+ return U.safeAbs(hash) % segments;
+ }
+
+ /**
+ *
+ */
+ private class PagePool {
+ /** Segment index. */
+ protected final int idx;
+
+ /** Direct memory region. */
+ protected final DirectMemoryRegion region;
+
+ /** */
+ protected long lastAllocatedIdxPtr;
+
+ /** Pointer to the address of the free page list. */
+ protected long freePageListPtr;
+
+ /** Pages base. */
+ protected long pagesBase;
+
+ /**
+ * @param idx Index.
+ * @param region Region
+ */
+ protected PagePool(int idx, DirectMemoryRegion region) {
+ this.idx = idx;
+ this.region = region;
+
+ long base = (region.address() + 7) & ~0x7;
+
+ freePageListPtr = base;
+
+ base += 8;
+
+ lastAllocatedIdxPtr = base;
+
+ base += 8;
+
+ // Align page start by
+ pagesBase = base;
+
+ GridUnsafe.putLong(freePageListPtr, INVALID_REL_PTR);
+ GridUnsafe.putLong(lastAllocatedIdxPtr, 1L);
+ }
+
+ /**
+ * Allocates a new free page.
+ *
+ * @param pageId Page ID to to initialize.
+ * @return Relative pointer to the allocated page.
+ * @throws GridOffHeapOutOfMemoryException If failed to allocate new free page.
+ */
+ private long borrowOrAllocateFreePage(long pageId) throws GridOffHeapOutOfMemoryException {
+ long relPtr = borrowFreePage();
+
+ return relPtr != INVALID_REL_PTR ? relPtr : allocateFreePage(pageId);
+ }
+
+ /**
+ * @return Relative pointer to a free page that was borrowed from the allocated pool.
+ */
+ private long borrowFreePage() {
+ while (true) {
+ long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr);
+
+ long freePageRelPtr = freePageRelPtrMasked & ADDRESS_MASK;
+
+ if (freePageRelPtr != INVALID_REL_PTR) {
+ long freePageAbsPtr = absolute(freePageRelPtr);
+
+ long nextFreePageRelPtr = GridUnsafe.getLong(freePageAbsPtr) & ADDRESS_MASK;
+
+ long cnt = ((freePageRelPtrMasked & COUNTER_MASK) + COUNTER_INC) & COUNTER_MASK;
+
+ if (GridUnsafe.compareAndSwapLong(null, freePageListPtr, freePageRelPtrMasked, nextFreePageRelPtr | cnt)) {
+ GridUnsafe.putLong(freePageAbsPtr, PAGE_MARKER);
+
+ return freePageRelPtr;
+ }
+ }
+ else
+ return INVALID_REL_PTR;
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @return Relative pointer of the allocated page.
+ * @throws GridOffHeapOutOfMemoryException If failed to allocate new free page.
+ */
+ private long allocateFreePage(long pageId) throws GridOffHeapOutOfMemoryException {
+ long limit = region.address() + region.size();
+
+ while (true) {
+ long lastIdx = GridUnsafe.getLong(lastAllocatedIdxPtr);
+
+ // Check if we have enough space to allocate a page.
+ if (pagesBase + (lastIdx + 1) * sysPageSize > limit)
+ return INVALID_REL_PTR;
+
+ if (GridUnsafe.compareAndSwapLong(null, lastAllocatedIdxPtr, lastIdx, lastIdx + 1)) {
+ long absPtr = pagesBase + lastIdx * sysPageSize;
+
+ assert (lastIdx & SEGMENT_INDEX_MASK) == 0L;
+
+ long relative = relative(lastIdx);
+
+ assert relative != INVALID_REL_PTR;
+
+ PageHeader.initNew(absPtr, relative);
+
+ rwLock.init(absPtr + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
+
+ return relative;
+ }
+ }
+ }
+
+ /**
+ * @param relPtr Relative pointer to free.
+ */
+ private void releaseFreePage(long relPtr) {
+ long absPtr = absolute(relPtr);
+
+ assert !PageHeader.isAcquired(absPtr) : "Release pinned page: " + PageHeader.fullPageId(absPtr);
+
+ while (true) {
+ long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr);
+
+ long freePageRelPtr = freePageRelPtrMasked & RELATIVE_PTR_MASK;
+
+ GridUnsafe.putLong(absPtr, freePageRelPtr);
+
+ if (GridUnsafe.compareAndSwapLong(null, freePageListPtr, freePageRelPtrMasked, relPtr))
+ return;
+ }
+ }
+
+ /**
+ * @param relativePtr Relative pointer.
+ * @return Absolute pointer.
+ */
+ long absolute(long relativePtr) {
+ int segIdx = (int)((relativePtr >> 40) & 0xFFFF);
+
+ assert segIdx == idx : "expected=" + idx + ", actual=" + segIdx;
+
+ long pageIdx = relativePtr & ~SEGMENT_INDEX_MASK;
+
+ long off = pageIdx * sysPageSize;
+
+ return pagesBase + off;
+ }
+
+ /**
+ * @param pageIdx Page index in the pool.
+ * @return Relative pointer.
+ */
+ private long relative(long pageIdx) {
+ return pageIdx | ((long)idx) << 40;
+ }
+
+ /**
+ * @return Max number of pages in the pool.
+ */
+ private int pages() {
+ return (int)((region.size() - (pagesBase - region.address())) / sysPageSize);
+ }
+ }
+
+ /**
+ *
+ */
+ private class Segment extends ReentrantReadWriteLock {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private static final double FULL_SCAN_THRESHOLD = 0.4;
+
+ /** Page ID to relative pointer map. */
+ private FullPageIdTable loadedPages;
+
+ /** */
+ private long acquiredPagesPtr;
+
+ /** */
+ private PagePool pool;
+
+ /** */
+ private long memPerTbl;
+
+ /** Pages marked as dirty since the last checkpoint. */
+ private Collection<FullPageId> dirtyPages = new GridConcurrentHashSet<>();
+
+ /** */
+ private volatile Collection<FullPageId> segCheckpointPages;
+
+ /** */
+ private final int maxDirtyPages;
+
+ /** */
+ private final Map<T2<Integer, Integer>, Integer> partTagMap = new HashMap<>();
+
+ /**
+ * @param region Memory region.
+ */
+ private Segment(int idx, DirectMemoryRegion region, int cpPoolPages) {
+ long totalMemory = region.size();
+
+ int pages = (int)(totalMemory / sysPageSize);
+
+ memPerTbl = requiredSegmentTableMemory(pages);
+
+ acquiredPagesPtr = region.address();
+
+ GridUnsafe.putIntVolatile(null, acquiredPagesPtr, 0);
+
+ loadedPages = new FullPageIdTable(region.address() + 8, memPerTbl, true);
+
+ DirectMemoryRegion poolRegion = region.slice(memPerTbl + 8);
+
+ pool = new PagePool(idx, poolRegion);
+
+ maxDirtyPages = Math.min(pool.pages() * 2 / 3, cpPoolPages);
+ }
+
+ /**
+ *
+ */
+ private boolean safeToUpdate() {
+ return dirtyPages.size() < maxDirtyPages;
+ }
+
+ /**
+ * @return Max number of pages this segment can allocate.
+ */
+ private int pages() {
+ return pool.pages();
+ }
+
+ /**
+ * @return Memory allocated for pages table.
+ */
+ private long tableSize() {
+ return memPerTbl;
+ }
+
+ /**
+ * @param absPtr Page absolute address to acquire.
+ */
+ private void acquirePage(long absPtr) {
+ PageHeader.acquirePage(absPtr);
+
+ updateAtomicInt(acquiredPagesPtr, 1);
+ }
+
+ /**
+ * @param absPtr Page absolute address to release.
+ */
+ private void releasePage(long absPtr) {
+ PageHeader.releasePage(absPtr);
+
+ updateAtomicInt(acquiredPagesPtr, -1);
+ }
+
+ /**
+ * @return Total number of acquired pages.
+ */
+ private int acquiredPages() {
+ return GridUnsafe.getInt(acquiredPagesPtr);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @return Page relative pointer.
+ */
+ private long borrowOrAllocateFreePage(long pageId) {
+ return pool.borrowOrAllocateFreePage(pageId);
+ }
+
+ /**
+ * Prepares a page for eviction, if needed.
+ *
+ * @param fullPageId Candidate page full ID.
+ * @param absPtr Absolute pointer of the page to evict.
+ * @return {@code True} if it is ok to evict this page, {@code false} if another page should be selected.
+ * @throws IgniteCheckedException If failed to write page to the underlying store during eviction.
+ */
+ private boolean prepareEvict(FullPageId fullPageId, long absPtr) throws IgniteCheckedException {
+ assert writeLock().isHeldByCurrentThread();
+
+ // Do not evict cache meta pages.
+ if (fullPageId.pageId() == storeMgr.metaPageId(fullPageId.cacheId()))
+ return false;
+
+ if (PageHeader.isAcquired(absPtr))
+ return false;
+
+ Collection<FullPageId> cpPages = segCheckpointPages;
+
+ if (isDirty(absPtr)) {
+ // Can evict a dirty page only if should be written by a checkpoint.
+ if (cpPages != null && cpPages.contains(fullPageId)) {
+ assert storeMgr != null;
+
+ flushDirtyPage.applyx(
+ fullPageId,
+ wrapPointer(absPtr + PAGE_OVERHEAD, pageSize()),
+ partTag(
+ fullPageId.cacheId(),
+ PageIdUtils.partId(fullPageId.pageId())
+ )
+ );
+
+ setDirty(fullPageId, absPtr, false, true);
+
+ cpPages.remove(fullPageId);
+
+ return true;
+ }
+
+ return false;
+ }
+ else
+ // Page was not modified, ok to evict.
+ return true;
+ }
+
+ /**
+ * Evict random oldest page from memory to storage.
+ *
+ * @return Relative address for evicted page.
+ * @throws IgniteCheckedException If failed to evict page.
+ */
+ private long evictPage() throws IgniteCheckedException {
+ assert getWriteHoldCount() > 0;
+
+ if (!pageEvictWarned) {
+ pageEvictWarned = true;
+
+ U.warn(log, "Page evictions started, this will affect storage performance (consider increasing " +
+ "MemoryConfiguration#setPageCacheSize).");
+ }
+
+ final ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ final int cap = loadedPages.capacity();
+
+ if (acquiredPages() >= loadedPages.size())
+ throw new IgniteOutOfMemoryException("Failed to evict page from segment (all pages are acquired).");
+
+ // With big number of random picked pages we may fall into infinite loop, because
+ // every time the same page may be found.
+ Set<Long> ignored = null;
+
+ long relEvictAddr = INVALID_REL_PTR;
+
+ int iterations = 0;
+
+ while (true) {
+ long cleanAddr = INVALID_REL_PTR;
+ long cleanTs = Long.MAX_VALUE;
+ long dirtyTs = Long.MAX_VALUE;
+ long dirtyAddr = INVALID_REL_PTR;
+
+ for (int i = 0; i < RANDOM_PAGES_EVICT_NUM; i++) {
+ ++iterations;
+
+ if (iterations > pool.pages() * FULL_SCAN_THRESHOLD)
+ break;
+
+ // We need to lookup for pages only in current segment for thread safety,
+ // so peeking random memory will lead to checking for found page segment.
+ // It's much faster to check available pages for segment right away.
+ EvictCandidate nearest = loadedPages.getNearestAt(rnd.nextInt(cap), INVALID_REL_PTR);
+
+ assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
+
+ long rndAddr = nearest.relativePointer();
+
+ int partTag = nearest.tag();
+
+ final long absPageAddr = absolute(rndAddr);
+
+ FullPageId fullId = PageHeader.fullPageId(absPageAddr);
+
+ // Check page mapping consistency.
+ assert fullId.equals(nearest.fullId()) : "Invalid page mapping [tableId=" + nearest.fullId() +
+ ", actual=" + fullId + ", nearest=" + nearest;
+
+ boolean outdated = partTag < partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId()));
+
+ if (outdated)
+ return refreshOutdatedPage(this, fullId.cacheId(), fullId.pageId(), true);
+
+ boolean pinned = PageHeader.isAcquired(absPageAddr);
+
+ boolean skip = ignored != null && ignored.contains(rndAddr);
+
+ if (relEvictAddr == rndAddr || pinned || skip) {
+ i--;
+
+ continue;
+ }
+
+ final long pageTs = PageHeader.readTimestamp(absPageAddr);
+
+ final boolean dirty = isDirty(absPageAddr);
+
+ if (pageTs < cleanTs && !dirty) {
+ cleanAddr = rndAddr;
+
+ cleanTs = pageTs;
+ }
+ else if (pageTs < dirtyTs && dirty) {
+ dirtyAddr = rndAddr;
+
+ dirtyTs = pageTs;
+ }
+
+ relEvictAddr = cleanAddr == INVALID_REL_PTR ? dirtyAddr : cleanAddr;
+ }
+
+ assert relEvictAddr != INVALID_REL_PTR;
+
+ final long absEvictAddr = absolute(relEvictAddr);
+
+ final FullPageId fullPageId = PageHeader.fullPageId(absEvictAddr);
+
+ if (!prepareEvict(fullPageId, absEvictAddr)) {
+ if (iterations > 10) {
+ if (ignored == null)
+ ignored = new HashSet<>();
+
+ ignored.add(relEvictAddr);
+ }
+
+ if (iterations > pool.pages() * FULL_SCAN_THRESHOLD)
+ return tryToFindSequentially(cap);
+
+ continue;
+ }
+
+ loadedPages.remove(
+ fullPageId.cacheId(),
+ PageIdUtils.effectivePageId(fullPageId.pageId()),
+ partTag(
+ fullPageId.cacheId(),
+ PageIdUtils.partId(fullPageId.pageId())
+ )
+ );
+
+ return relEvictAddr;
+ }
+ }
+
+ /**
+ * Will scan all segment pages to find one to evict it
+ *
+ * @param cap Capacity.
+ */
+ private long tryToFindSequentially(int cap) throws IgniteCheckedException {
+ assert getWriteHoldCount() > 0;
+
+ long prevAddr = INVALID_REL_PTR;
+ int pinnedCnt = 0;
+ int failToPrepare = 0;
+
+ for (int i = 0; i < cap; i++) {
+ final EvictCandidate nearest = loadedPages.getNearestAt(i, INVALID_REL_PTR);
+
+ assert nearest != null && nearest.relativePointer() != INVALID_REL_PTR;
+
+ final long addr = nearest.relativePointer();
+
+ int partTag = nearest.tag();
+
+ final long absPageAddr = absolute(addr);
+
+ FullPageId fullId = PageHeader.fullPageId(absPageAddr);
+
+ if (partTag < partTag(fullId.cacheId(), PageIdUtils.partId(fullId.pageId())))
+ return refreshOutdatedPage(this, fullId.cacheId(), fullId.pageId(), true);
+
+ boolean pinned = PageHeader.isAcquired(absPageAddr);
+
+ if (pinned)
+ pinnedCnt++;
+
+ if (addr == prevAddr || pinned)
+ continue;
+
+ final long absEvictAddr = absolute(addr);
+
+ final FullPageId fullPageId = PageHeader.fullPageId(absEvictAddr);
+
+ if (prepareEvict(fullPageId, absEvictAddr)) {
+ loadedPages.remove(
+ fullPageId.cacheId(),
+ PageIdUtils.effectivePageId(fullPageId.pageId()),
+ partTag(
+ fullPageId.cacheId(),
+ PageIdUtils.partId(fullPageId.pageId())
+ )
+ );
+
+ return addr;
+ }
+ else
+ failToPrepare++;
+
+ prevAddr = addr;
+ }
+
+ throw new IgniteOutOfMemoryException("Failed to find a page for eviction [segmentCapacity=" + cap +
+ ", loaded=" + loadedPages.size() +
+ ", maxDirtyPages=" + maxDirtyPages +
+ ", dirtyPages=" + dirtyPages.size() +
+ ", cpPages=" + (segCheckpointPages == null ? 0 : segCheckpointPages.size()) +
+ ", pinnedInSegment=" + pinnedCnt +
+ ", failedToPrepare=" + failToPrepare +
+ ']');
+ }
+
+ /**
+ * Delegate to the corresponding page pool.
+ *
+ * @param relPtr Relative pointer.
+ * @return Absolute pointer.
+ */
+ private long absolute(long relPtr) {
+ return pool.absolute(relPtr);
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param partId Partition ID.
+ * @return Partition tag.
+ */
+ private int partTag(int cacheId, int partId) {
+ assert getReadHoldCount() > 0 || getWriteHoldCount() > 0;
+
+ Integer tag = partTagMap.get(new T2<>(cacheId, partId));
+
+ if (tag == null)
+ return 1;
+
+ return tag;
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param partId Partition ID.
+ * @return New partition tag.
+ */
+ private int incrementPartTag(int cacheId, int partId) {
+ assert getWriteHoldCount() > 0;
+
+ T2<Integer, Integer> t = new T2<>(cacheId, partId);
+
+ Integer tag = partTagMap.get(t);
+
+ if (tag == null) {
+ partTagMap.put(t, 2);
+
+ return 2;
+ }
+ else if (tag == Integer.MAX_VALUE) {
+ U.warn(log, "Partition tag overflow [cacheId=" + cacheId + ", partId=" + partId + "]");
+
+ partTagMap.put(t, 0);
+
+ return 0;
+ }
+ else {
+ partTagMap.put(t, tag + 1);
+
+ return tag + 1;
+ }
+ }
+
+ /**
+ * @param grpId Cache group id.
+ */
+ private void resetPartTags(int grpId) {
+ assert getWriteHoldCount() > 0;
+
+ Iterator<T2<Integer, Integer>> iter = partTagMap.keySet().iterator();
+
+ while (iter.hasNext()) {
+ T2<Integer, Integer> t = iter.next();
+
+ if (t.get1() == grpId)
+ iter.remove();
+ }
+ }
+ }
+
+ /**
+ * Gets an estimate for the amount of memory required to store the given number of page IDs
+ * in a segment table.
+ *
+ * @param pages Number of pages to store.
+ * @return Memory size estimate.
+ */
+ private static long requiredSegmentTableMemory(int pages) {
+ return FullPageIdTable.requiredMemory(pages) + 8;
+ }
+
+ /**
+ * @param ptr Pointer to update.
+ * @param delta Delta.
+ */
+ private static int updateAtomicInt(long ptr, int delta) {
+ while (true) {
+ int old = GridUnsafe.getInt(ptr);
+
+ int updated = old + delta;
+
+ if (GridUnsafe.compareAndSwapInt(null, ptr, old, updated))
+ return updated;
+ }
+ }
+
+ /**
+ * @param ptr Pointer to update.
+ * @param delta Delta.
+ */
+ private static long updateAtomicLong(long ptr, long delta) {
+ while (true) {
+ long old = GridUnsafe.getLong(ptr);
+
+ long updated = old + delta;
+
+ if (GridUnsafe.compareAndSwapLong(null, ptr, old, updated))
+ return updated;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class PageHeader {
+ /**
+ * @param absPtr Absolute pointer to initialize.
+ * @param relative Relative pointer to write.
+ */
+ private static void initNew(long absPtr, long relative) {
+ relative(absPtr, relative);
+
+ tempBufferPointer(absPtr, INVALID_REL_PTR);
+
+ GridUnsafe.putLong(absPtr, PAGE_MARKER);
+ GridUnsafe.putInt(absPtr + PAGE_PIN_CNT_OFFSET, 0);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @return Dirty flag.
+ */
+ private static boolean dirty(long absPtr) {
+ return flag(absPtr, DIRTY_FLAG);
+ }
+
+ /**
+ * @param absPtr Page absolute pointer.
+ * @param dirty Dirty flag.
+ * @return Previous value of dirty flag.
+ */
+ private static boolean dirty(long absPtr, boolean dirty) {
+ return flag(absPtr, DIRTY_FLAG, dirty);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @return Dirty flag.
+ */
+ private static boolean tempDirty(long absPtr) {
+ return flag(absPtr, TMP_DIRTY_FLAG);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @param tmpDirty Temp dirty flag.
+ * @return Previous value of temp dirty flag.
+ */
+ private static boolean tempDirty(long absPtr, boolean tmpDirty) {
+ return flag(absPtr, TMP_DIRTY_FLAG, tmpDirty);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ * @param flag Flag mask.
+ * @return Flag value.
+ */
+ private static boolean flag(long absPtr, long flag) {
+ assert (flag & 0xFFFFFFFFFFFFFFL) == 0;
+ assert Long.bitCount(flag) == 1;
+
+ long relPtrWithFlags = GridUnsafe.getLong(absPtr + RELATIVE_PTR_OFFSET);
+
+ return (relPtrWithFlags & flag) != 0;
+ }
+
+ /**
+ * Sets flag.
+ *
+ * @param absPtr Absolute pointer.
+ * @param flag Flag mask.
+ * @param set New flag value.
+ * @return Previous flag value.
+ */
+ private static boolean flag(long absPtr, long flag, boolean set) {
+ assert (flag & 0xFFFFFFFFFFFFFFL) == 0;
+ assert Long.bitCount(flag) == 1;
+
+ long relPtrWithFlags = GridUnsafe.getLong(absPtr + RELATIVE_PTR_OFFSET);
+
+ boolean was = (relPtrWithFlags & flag) != 0;
+
+ if (set)
+ relPtrWithFlags |= flag;
+ else
+ relPtrWithFlags &= ~flag;
+
+ GridUnsafe.putLong(absPtr + RELATIVE_PTR_OFFSET, relPtrWithFlags);
+
+ return was;
+ }
+
+ /**
+ * @param absPtr Page pointer.
+ * @return If page is pinned.
+ */
+ private static boolean isAcquired(long absPtr) {
+ return GridUnsafe.getInt(absPtr + PAGE_PIN_CNT_OFFSET) > 0;
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ */
+ private static void acquirePage(long absPtr) {
+ updateAtomicInt(absPtr + PAGE_PIN_CNT_OFFSET, 1);
+ }
+
+ /**
+ * @param absPtr Absolute pointer.
+ */
+ private static int releasePage(long absPtr) {
+ return updateAtomicInt(absPtr + PAGE_PIN_CNT_OFFSET, -1);
+ }
+
+ /**
+ * Reads relative pointer from the page at the given absolute position.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @return Relative pointer written to the page.
+ */
+ private static long readRelative(long absPtr) {
+ return GridUnsafe.getLong(absPtr + RELATIVE_PTR_OFFSET) & RELATIVE_PTR_MASK;
+ }
+
+ /**
+ * Writes relative pointer to the page at the given absolute position.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @param relPtr Relative pointer to write.
+ */
+ private static void relative(long absPtr, long relPtr) {
+ GridUnsafe.putLong(absPtr + RELATIVE_PTR_OFFSET, relPtr & RELATIVE_PTR_MASK);
+ }
+
+ /**
+ * Volatile write for current timestamp to page in {@code absAddr} address.
+ *
+ * @param absPtr Absolute page address.
+ */
+ private static void writeTimestamp(final long absPtr, long tstamp) {
+ tstamp >>= 8;
+
+ GridUnsafe.putLongVolatile(null, absPtr, (tstamp << 8) | 0x01);
+ }
+
+ /**
+ * Read for timestamp from page in {@code absAddr} address.
+ *
+ * @param absPtr Absolute page address.
+ * @return Timestamp.
+ */
+ private static long readTimestamp(final long absPtr) {
+ long markerAndTs = GridUnsafe.getLong(absPtr);
+
+ // Clear last byte as it is occupied by page marker.
+ return markerAndTs & ~0xFF;
+ }
+
+ /**
+ * @param absPtr Page absolute pointer.
+ * @param tmpRelPtr Temp buffer relative pointer.
+ */
+ private static void tempBufferPointer(long absPtr, long tmpRelPtr) {
+ GridUnsafe.putLong(absPtr + PAGE_TMP_BUF_OFFSET, tmpRelPtr);
+ }
+
+ /**
+ * @param absPtr Page absolute pointer.
+ * @return Temp buffer relative pointer.
+ */
+ private static long tempBufferPointer(long absPtr) {
+ return GridUnsafe.getLong(absPtr + PAGE_TMP_BUF_OFFSET);
+ }
+
+ /**
+ * Reads page ID from the page at the given absolute position.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @return Page ID written to the page.
+ */
+ private static long readPageId(long absPtr) {
+ return GridUnsafe.getLong(absPtr + PAGE_ID_OFFSET);
+ }
+
+ /**
+ * Writes page ID to the page at the given absolute position.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @param pageId Page ID to write.
+ */
+ private static void pageId(long absPtr, long pageId) {
+ GridUnsafe.putLong(absPtr + PAGE_ID_OFFSET, pageId);
+ }
+
+ /**
+ * Reads cache ID from the page at the given absolute pointer.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @return Cache ID written to the page.
+ */
+ private static int readPageCacheId(final long absPtr) {
+ return GridUnsafe.getInt(absPtr + PAGE_CACHE_ID_OFFSET);
+ }
+
+ /**
+ * Writes cache ID from the page at the given absolute pointer.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @param cacheId Cache ID to write.
+ */
+ private static void pageCacheId(final long absPtr, final int cacheId) {
+ GridUnsafe.putInt(absPtr + PAGE_CACHE_ID_OFFSET, cacheId);
+ }
+
+ /**
+ * Reads page ID and cache ID from the page at the given absolute pointer.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @return Full page ID written to the page.
+ */
+ private static FullPageId fullPageId(final long absPtr) {
+ return new FullPageId(readPageId(absPtr), readPageCacheId(absPtr));
+ }
+
+ /**
+ * Writes page ID and cache ID from the page at the given absolute pointer.
+ *
+ * @param absPtr Absolute memory pointer to the page header.
+ * @param fullPageId Full page ID to write.
+ */
+ private static void fullPageId(final long absPtr, final FullPageId fullPageId) {
+ pageId(absPtr, fullPageId.pageId());
+
+ pageCacheId(absPtr, fullPageId.cacheId());
+ }
+ }
+
+ /**
+ *
+ */
+ private static class ClearSegmentRunnable implements Runnable {
+ /** */
+ private Segment seg;
+
+ /** */
+ private GridPredicate3<Integer, Long, Integer> clearPred;
+
+ /** */
+ private CountDownFuture doneFut;
+
+ /** */
+ private int pageSize;
+
+ /** */
+ private boolean rmvDirty;
+
+ /**
+ * @param seg Segment.
+ * @param clearPred Clear predicate.
+ * @param doneFut Completion future.
+ */
+ private ClearSegmentRunnable(
+ Segment seg,
+ GridPredicate3<Integer, Long, Integer> clearPred,
+ boolean rmvDirty,
+ CountDownFuture doneFut,
+ int pageSize
+ ) {
+ this.seg = seg;
+ this.clearPred = clearPred;
+ this.rmvDirty = rmvDirty;
+ this.doneFut = doneFut;
+ this.pageSize = pageSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ int cap = seg.loadedPages.capacity();
+
+ int chunkSize = 1000;
+ GridLongList ptrs = new GridLongList(chunkSize);
+
+ try {
+ for (int base = 0; base < cap; ) {
+ int boundary = Math.min(cap, base + chunkSize);
+
+ seg.writeLock().lock();
+
+ try {
+ while (base < boundary) {
+ long ptr = seg.loadedPages.clearAt(base, clearPred, INVALID_REL_PTR);
+
+ if (ptr != INVALID_REL_PTR)
+ ptrs.add(ptr);
+
+ base++;
+ }
+ }
+ finally {
+ seg.writeLock().unlock();
+ }
+
+ for (int i = 0; i < ptrs.size(); i++) {
+ long relPtr = ptrs.get(i);
+
+ long absPtr = seg.pool.absolute(relPtr);
+
+ if (rmvDirty) {
+ FullPageId fullId = PageHeader.fullPageId(absPtr);
+
+ seg.dirtyPages.remove(fullId);
+ }
+
+ GridUnsafe.setMemory(absPtr + PAGE_OVERHEAD, pageSize, (byte)0);
+
+ seg.pool.releaseFreePage(relPtr);
+ }
+
+ ptrs.clear();
+ }
+
+ doneFut.onDone((Void)null);
+ }
+ catch (Throwable e) {
+ doneFut.onDone(e);
+ }
+ }
+ }
+}
[18/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToActiveCluster.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToActiveCluster.java
new file mode 100644
index 0000000..6ba3a20
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToActiveCluster.java
@@ -0,0 +1,356 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join;
+
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.AbstractNodeJoinTemplate;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class JoinInActiveNodeToActiveCluster extends AbstractNodeJoinTemplate {
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(true),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheOnlySystem()
+ ).nodeConfiguration(
+ cfg(name(3)).setActiveOnStart(false)
+ ).afterNodeJoin(
+ b.checkCacheOnlySystem()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheOnlySystem()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(true),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheOnlySystem()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheNotEmpty()
+ ).nodeConfiguration(
+ cfg(name(3)).setActiveOnStart(false)
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheNotEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(atomicCfg()),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(3, desc.size());
+
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(3, caches.size());
+ }
+ }
+ }
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(transactionCfg())
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ // Server node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinWithOutConfiguration() throws Exception {
+ withOutConfigurationTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
+ staticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationInCluster() throws Exception {
+ staticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
+ staticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ staticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ // Client node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientWithOutConfiguration() throws Exception {
+ joinClientWithOutConfigurationTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
+ joinClientStaticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
+ joinClientStaticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
+ return withOutConfigurationTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
+ return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
+ return staticCacheConfigurationInClusterTemplate()
+ .nodeConfiguration(setClient)
+ .afterNodeJoin(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient()) {
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+ else {
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+ }
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ }
+ }).setEnd(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient()) {
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+ else {
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+ }
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ }
+ });
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
+ return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return staticCacheConfigurationDifferentOnBothTemplate()
+ .nodeConfiguration(setClient)
+ .afterNodeJoin(
+ new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 4; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertEquals(4, caches.size());
+ else
+ Assert.assertEquals(3, caches.size());
+ }
+ }
+ }
+ ).setEnd(
+ new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 4; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertEquals(4, caches.size());
+ else
+ Assert.assertEquals(3, caches.size());
+ }
+ }
+ }
+ );
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToInActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToInActiveCluster.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToInActiveCluster.java
new file mode 100644
index 0000000..244001b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinInActiveNodeToInActiveCluster.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join;
+
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.AbstractNodeJoinTemplate;
+
+/**
+ *
+ */
+public class JoinInActiveNodeToInActiveCluster extends AbstractNodeJoinTemplate {
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(false),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3)).setActiveOnStart(false)
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheOnlySystem()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(false),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3)).setActiveOnStart(false)
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(transactionCfg()),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(atomicCfg())
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ // Server node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinWithOutConfiguration() throws Exception {
+ withOutConfigurationTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
+ staticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationInCluster() throws Exception {
+ staticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
+ staticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ staticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ // Client node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientWithOutConfiguration() throws Exception {
+ joinClientWithOutConfigurationTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
+ joinClientStaticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
+ joinClientStaticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
+ return withOutConfigurationTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
+ return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
+ return staticCacheConfigurationInClusterTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
+ return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return staticCacheConfigurationDifferentOnBothTemplate().nodeConfiguration(setClient);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java
new file mode 100644
index 0000000..54087ba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.persistence;
+
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.JoinActiveNodeToActiveCluster;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.AbstractNodeJoinTemplate;
+
+/**
+ *
+ */
+public class JoinActiveNodeToActiveClusterWithPersistence extends JoinActiveNodeToActiveCluster {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration cfg(String name) throws Exception {
+ return persistentCfg(super.cfg(name));
+ }
+
+ private AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder persistent(AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder b) {
+ b.afterClusterStarted(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
+ AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder b = persistent(super.withOutConfigurationTemplate());
+
+ b.afterActivate(b.checkCacheOnlySystem());
+
+ return b;
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
+ AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder b = persistent(super.joinClientWithOutConfigurationTemplate());
+
+ b.afterActivate(b.checkCacheOnlySystem());
+
+ return b;
+ }
+
+ @Override public void testJoinWithOutConfiguration() throws Exception {
+ withOutConfigurationTemplate().execute();
+ }
+
+ @Override public void testJoinClientWithOutConfiguration() throws Exception {
+ joinClientWithOutConfigurationTemplate().execute();
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationOnJoinTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationInClusterTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationSameOnBothTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationDifferentOnBothTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
+ return persistent(super.joinClientStaticCacheConfigurationOnJoinTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
+ return persistent(super.joinClientStaticCacheConfigurationInClusterTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return persistent(super.joinClientStaticCacheConfigurationDifferentOnBothTemplate());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java
new file mode 100644
index 0000000..305255c
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.persistence;
+
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.JoinActiveNodeToInActiveCluster;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class JoinActiveNodeToInActiveClusterWithPersistence extends JoinActiveNodeToInActiveCluster {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration cfg(String name) throws Exception {
+ return persistentCfg(super.cfg(name));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java
new file mode 100644
index 0000000..a138d8d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.persistence;
+
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.JoinInActiveNodeToActiveCluster;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.AbstractNodeJoinTemplate;
+
+/**
+ *
+ */
+public class JoinInActiveNodeToActiveClusterWithPersistence extends JoinInActiveNodeToActiveCluster {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration cfg(String name) throws Exception {
+ return persistentCfg(super.cfg(name));
+ }
+
+ private AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder persistent(AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder b) {
+ b.afterClusterStarted(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
+ AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder b = persistent(super.withOutConfigurationTemplate());
+
+ b.afterActivate(b.checkCacheOnlySystem());
+
+ return b;
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
+ AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder b = persistent(super.joinClientWithOutConfigurationTemplate());
+
+ b.afterActivate(b.checkCacheOnlySystem());
+
+ return b;
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception{
+ return persistent(super.joinClientStaticCacheConfigurationInClusterTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return persistent(super.joinClientStaticCacheConfigurationDifferentOnBothTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationOnJoinTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationInClusterTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationSameOnBothTemplate());
+ }
+
+ @Override public AbstractNodeJoinTemplate.JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return persistent(super.staticCacheConfigurationDifferentOnBothTemplate());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java
new file mode 100644
index 0000000..b3db10d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.persistence;
+
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.join.JoinInActiveNodeToInActiveCluster;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class JoinInActiveNodeToInActiveClusterWithPersistence extends JoinInActiveNodeToInActiveCluster {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration cfg(String name) throws Exception {
+ return persistentCfg(super.cfg(name));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
new file mode 100644
index 0000000..eec267b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.reconnect;
+
+import com.google.common.collect.Sets;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.events.Event;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
+import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.Nullable;
+import org.junit.Assert;
+
+import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
+
+/**
+ *
+ */
+public abstract class IgniteAbstractStandByClientReconnectTest extends GridCommonAbstractTest {
+
+ private static final TcpDiscoveryVmIpFinder vmIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ private static final TcpDiscoveryVmIpFinder clientIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ protected final String node1 = "node1";
+ protected final String node2 = "node2";
+ protected final String nodeClient = "nodeClient";
+
+ protected final String ccfg1staticName = "cache1static";
+ protected final String ccfg2staticName = "cache2static";
+ protected final String ccfg3staticName = "cache3static";
+
+ protected final String ccfg1staticWithFilterName = "ccfg1staticWithFilter";
+ protected final String ccfg2staticWithFilterName = "ccfg2staticWithFilter";
+ protected final String ccfg3staticWithFilterName = "ccfg3staticWithFilter";
+
+ protected final String ccfgDynamicName = "ccfgDynamic";
+ protected final String ccfgDynamicWithFilterName = "ccfgDynamicWithFilter";
+
+ protected final CacheConfiguration ccfg1static = new CacheConfiguration(ccfg1staticName);
+ protected final CacheConfiguration ccfg2static = new CacheConfiguration(ccfg2staticName);
+ protected final CacheConfiguration ccfg3static = new CacheConfiguration(ccfg3staticName);
+
+ protected final CacheConfiguration ccfg1staticWithFilter =
+ new CacheConfiguration(ccfg1staticWithFilterName).setNodeFilter(new FilterNode(node2));
+
+ protected final CacheConfiguration ccfg2staticWithFilter =
+ new CacheConfiguration(ccfg2staticWithFilterName).setNodeFilter(new FilterNode(nodeClient));
+
+ protected final CacheConfiguration ccfg3staticWithFilter =
+ new CacheConfiguration(ccfg3staticWithFilterName).setNodeFilter(new FilterNode(node1));
+
+ protected final CacheConfiguration ccfgDynamic = new CacheConfiguration(ccfgDynamicName);
+
+ protected final CacheConfiguration ccfgDynamicWithFilter =
+ new CacheConfiguration(ccfgDynamicWithFilterName).setNodeFilter(new FilterNode(node2));
+
+ protected final Set<String> staticCacheNames = Sets.newHashSet(
+ ccfg1staticName, ccfg2staticName, ccfg3staticName,
+ ccfg1staticWithFilterName, ccfg2staticWithFilterName, ccfg3staticWithFilterName
+ );
+
+ protected final Set<String> allCacheNames = Sets.newHashSet(
+ ccfg1staticName, ccfg2staticName, ccfg3staticName,
+ ccfg1staticWithFilterName, ccfg2staticWithFilterName, ccfg3staticWithFilterName,
+ ccfgDynamicName, ccfgDynamicWithFilterName
+ );
+
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(name);
+
+ if (!nodeClient.equals(name))
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(vmIpFinder));
+ else {
+ clientIpFinder.setAddresses(Collections.singletonList("127.0.0.1:47501"));
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(clientIpFinder));
+ }
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+ cfg.setConsistentId(name);
+
+ return cfg;
+ }
+
+ protected void addDisconnectListener(
+ final CountDownLatch disconnectedLatch,
+ final CountDownLatch reconnectedLatch
+ ) {
+ grid(nodeClient).events().localListen(new IgnitePredicate<Event>() {
+ @Override public boolean apply(Event event) {
+ switch (event.type()) {
+ case EventType.EVT_CLIENT_NODE_DISCONNECTED:
+ info("Client disconnected");
+
+ disconnectedLatch.countDown();
+
+ break;
+ case EventType.EVT_CLIENT_NODE_RECONNECTED:
+ info("Client reconnected");
+
+ reconnectedLatch.countDown();
+ }
+
+ return true;
+ }
+ }, EventType.EVT_CLIENT_NODE_DISCONNECTED, EventType.EVT_CLIENT_NODE_RECONNECTED);
+ }
+
+ protected void checkDescriptors(IgniteEx ig, Set<String> cacheNames) {
+ Collection<DynamicCacheDescriptor> descs = ig.context().cache().cacheDescriptors().values();
+
+ assertEquals("Node name: " + ig.name(), cacheNames.size() + 2, descs.size());
+
+ int systemCnt = 0;
+
+ for (DynamicCacheDescriptor desc : descs)
+ if (!CU.isSystemCache(desc.cacheName()))
+ assertTrue(desc.cacheName(), cacheNames.contains(desc.cacheName()));
+ else
+ systemCnt++;
+
+ assertEquals(2, systemCnt);
+ }
+
+ protected void startNodes(CountDownLatch activateLatch) throws Exception {
+ IgniteConfiguration cfg1 = getConfiguration(node1)
+ .setCacheConfiguration(ccfg1static, ccfg1staticWithFilter);
+
+ IgniteConfiguration cfg2 = getConfiguration(node2)
+ .setCacheConfiguration(ccfg2static, ccfg2staticWithFilter);
+
+ IgniteConfiguration cfg3 = getConfiguration(nodeClient)
+ .setCacheConfiguration(ccfg3static, ccfg3staticWithFilter);
+
+ if (activateLatch != null)
+ cfg3.setDiscoverySpi(
+ new AwaitTcpDiscoverySpi(activateLatch)
+ .setIpFinder(clientIpFinder)
+ );
+
+ cfg3.setClientMode(true);
+
+ IgniteEx ig1 = startGrid(cfg1);
+ IgniteEx ig2 = startGrid(cfg2);
+ IgniteEx client = startGrid(cfg3);
+ }
+
+ protected void checkStaticCaches() {
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ Assert.assertNotNull(ig1.cache(ccfg1staticName));
+ Assert.assertNotNull(ig1.cache(ccfg2staticName));
+ Assert.assertNotNull(ig1.cache(ccfg3staticName));
+
+ Assert.assertNotNull(ig1.cache(ccfg1staticWithFilterName));
+ Assert.assertNotNull(ig1.cache(ccfg2staticWithFilterName));
+
+ Assert.assertNotNull(ig2.cache(ccfg1staticName));
+ Assert.assertNotNull(ig2.cache(ccfg2staticName));
+ Assert.assertNotNull(ig2.cache(ccfg3staticName));
+
+ Assert.assertNotNull(ig2.cache(ccfg3staticWithFilterName));
+ Assert.assertNotNull(ig2.cache(ccfg2staticWithFilterName));
+
+ Assert.assertNotNull(client.cache(ccfg1staticName));
+ Assert.assertNotNull(client.cache(ccfg2staticName));
+ Assert.assertNotNull(client.cache(ccfg3staticName));
+
+ Assert.assertNotNull(client.cache(ccfg3staticWithFilterName));
+ Assert.assertNotNull(client.cache(ccfg1staticWithFilterName));
+ }
+
+ protected void checkAllCaches() {
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ checkStaticCaches();
+
+ Assert.assertNotNull(ig1.cache(ccfgDynamicName));
+ Assert.assertNotNull(ig1.cache(ccfgDynamicWithFilterName));
+
+ Assert.assertNotNull(ig2.cache(ccfgDynamicName));
+
+ Assert.assertNotNull(client.cache(ccfgDynamicName));
+ Assert.assertNotNull(client.cache(ccfgDynamicWithFilterName));
+ }
+
+ protected void checkOnlySystemCaches() {
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ Assert.assertNull(ig1.cache(ccfg1staticName));
+ Assert.assertNull(ig1.cache(ccfg2staticName));
+ Assert.assertNull(ig1.cache(ccfg3staticName));
+
+ Assert.assertNull(ig1.cache(ccfg1staticWithFilterName));
+ Assert.assertNull(ig1.cache(ccfg2staticWithFilterName));
+
+ Assert.assertNull(ig2.cache(ccfg1staticName));
+ Assert.assertNull(ig2.cache(ccfg2staticName));
+ Assert.assertNull(ig2.cache(ccfg3staticName));
+
+ Assert.assertNull(ig2.cache(ccfg3staticWithFilterName));
+ Assert.assertNull(ig2.cache(ccfg2staticWithFilterName));
+
+ Assert.assertNull(client.cache(ccfg1staticName));
+ Assert.assertNull(client.cache(ccfg2staticName));
+ Assert.assertNull(client.cache(ccfg3staticName));
+
+ Assert.assertNull(client.cache(ccfg3staticWithFilterName));
+ Assert.assertNull(client.cache(ccfg1staticWithFilterName));
+
+ checkDescriptors(ig1,Collections.<String>emptySet());
+ checkDescriptors(ig2,Collections.<String>emptySet());
+ checkDescriptors(client, Collections.<String>emptySet());
+ }
+
+ private static class FilterNode implements IgnitePredicate<ClusterNode> {
+
+ private final String consistentId;
+
+ private FilterNode(String id) {
+ consistentId = id;
+ }
+
+ @Override public boolean apply(ClusterNode node) {
+ return !consistentId.equals(node.consistentId());
+ }
+ }
+
+ private static class AwaitTcpDiscoverySpi extends TcpDiscoverySpi {
+
+ private final CountDownLatch latch;
+
+ private AwaitTcpDiscoverySpi(CountDownLatch latch) {
+ this.latch = latch;
+ }
+
+ @Override public void setListener(@Nullable DiscoverySpiListener lsnr) {
+ super.setListener(new AwaitDiscoverySpiListener(latch, lsnr));
+ }
+ }
+
+ private static class AwaitDiscoverySpiListener implements DiscoverySpiListener {
+
+ private final CountDownLatch latch;
+
+ private final DiscoverySpiListener delegate;
+
+ private AwaitDiscoverySpiListener(
+ CountDownLatch latch,
+ DiscoverySpiListener delegate
+ ) {
+ this.latch = latch;
+ this.delegate = delegate;
+ }
+
+ @Override public void onLocalNodeInitialized(ClusterNode locNode) {
+ delegate.onLocalNodeInitialized(locNode);
+ }
+
+ @Override public void onDiscovery(
+ int type,
+ long topVer,
+ ClusterNode node,
+ Collection<ClusterNode> topSnapshot,
+ @Nullable Map<Long, Collection<ClusterNode>> topHist,
+ @Nullable DiscoverySpiCustomMessage data
+ ) {
+ delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, data);
+
+ if (type == EVT_CLIENT_NODE_DISCONNECTED)
+ try {
+ System.out.println("Await cluster change state");
+
+ latch.await();
+ }
+ catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
+ }
+
+ @Override protected void afterTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectTest.java
new file mode 100644
index 0000000..02f7d00
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.reconnect;
+
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.internal.IgniteEx;
+
+/**
+ *
+ */
+public class IgniteStandByClientReconnectTest extends IgniteAbstractStandByClientReconnectTest {
+
+ public void testActiveClientReconnectToActiveCluster() throws Exception {
+ CountDownLatch activateLatch = new CountDownLatch(1);
+
+ startNodes(activateLatch);
+
+ info(">>>> star grid");
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ client.active(true);
+
+ info(">>>> activate grid");
+
+ checkDescriptors(ig1, staticCacheNames);
+ checkDescriptors(ig2, staticCacheNames);
+ checkDescriptors(client, staticCacheNames);
+
+ checkStaticCaches();
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ info(">>>> stop servers");
+
+ stopGrid(node2);
+
+ disconnectedLatch.await();
+
+ ig2 = startGrid(getConfiguration(node2));
+
+ info(">>>> activate new servers");
+
+ ig1.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+
+ activateLatch.countDown();
+
+ info(">>>> reconnect client");
+
+ reconnectedLatch.await();
+
+ info(">>>> client reconnected");
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ checkAllCaches();
+ }
+
+ public void testActiveClientReconnectToInActiveCluster() throws Exception {
+ CountDownLatch activateLatch = new CountDownLatch(1);
+
+ startNodes(activateLatch);
+
+ info(">>>> star grid");
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ info(">>>> activate grid");
+
+ client.active(true);
+
+ checkStaticCaches();
+
+ checkDescriptors(ig1, staticCacheNames);
+ checkDescriptors(ig2, staticCacheNames);
+ checkDescriptors(client, staticCacheNames);
+
+ info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ info(">>>> stop " + node2);
+
+ stopGrid(node2);
+
+ disconnectedLatch.await();
+
+ ig1.active(false);
+
+ activateLatch.countDown();
+
+ info(">>>> restart " + node2);
+
+ ig2 = startGrid(getConfiguration(node2));
+
+ reconnectedLatch.await();
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ client.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ checkAllCaches();
+ }
+
+ public void testInActiveClientReconnectToActiveCluster() throws Exception {
+ CountDownLatch activateLatch = new CountDownLatch(1);
+
+ startNodes(activateLatch);
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ stopGrid(node2);
+
+ disconnectedLatch.await();
+
+ ig2 = startGrid(getConfiguration(node2));
+
+ ig1.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+
+ checkDescriptors(ig1, staticCacheNames);
+ checkDescriptors(ig2, staticCacheNames);
+
+ activateLatch.countDown();
+
+ reconnectedLatch.await();
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkDescriptors(ig1, staticCacheNames);
+ checkDescriptors(ig2, staticCacheNames);
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ checkAllCaches();
+ }
+
+ public void testInActiveClientReconnectToInActiveCluster() throws Exception {
+ startNodes(null);
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ stopGrid(node2);
+
+ disconnectedLatch.await();
+
+ ig2 = startGrid(getConfiguration(node2));
+
+ reconnectedLatch.await();
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ client.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkStaticCaches();
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ checkAllCaches();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
new file mode 100644
index 0000000..2bcc177
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/reconnect/IgniteStandByClientReconnectToNewClusterTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.reconnect;
+
+import com.google.common.collect.Sets;
+import java.util.Collections;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.internal.IgniteEx;
+
+/**
+ *
+ */
+public class IgniteStandByClientReconnectToNewClusterTest extends IgniteAbstractStandByClientReconnectTest {
+
+ public void testActiveClientReconnectToActiveCluster() throws Exception {
+ CountDownLatch activateLatch = new CountDownLatch(1);
+
+ startNodes(activateLatch);
+
+ info(">>>> star grid");
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ info(">>>> activate grid");
+
+ client.active(true);
+
+ checkDescriptors(ig1, staticCacheNames);
+ checkDescriptors(ig2, staticCacheNames);
+ checkDescriptors(client, staticCacheNames);
+
+ checkStaticCaches();
+
+ info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ info(">>>> stop servers");
+
+ stopGrid(node1);
+ stopGrid(node2);
+
+ disconnectedLatch.await();
+
+ ig1 = startGrid(getConfiguration(node1));
+ ig2 = startGrid(getConfiguration(node2));
+
+ info(">>>> activate new servers");
+
+ ig1.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+
+ activateLatch.countDown();
+
+ info(">>>> reconnect client");
+
+ reconnectedLatch.await();
+
+ info(">>>> client reconnected");
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkAllCaches();
+ }
+
+ public void testActiveClientReconnectToInActiveCluster() throws Exception {
+ startNodes(null);
+
+ info(">>>> star grid");
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ info(">>>> activate grid");
+
+ client.active(true);
+
+ checkStaticCaches();
+
+ checkDescriptors(ig1, staticCacheNames);
+ checkDescriptors(ig2, staticCacheNames);
+ checkDescriptors(client, staticCacheNames);
+
+ info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ checkDescriptors(ig1, allCacheNames);
+ checkDescriptors(ig2, allCacheNames);
+ checkDescriptors(client, allCacheNames);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ info(">>>> stop servers");
+
+ stopGrid(node1);
+ stopGrid(node2);
+
+ assertTrue(client.active());
+
+ System.out.println("Await disconnected");
+
+ disconnectedLatch.await();
+
+ ig1 = startGrid(getConfiguration("node1"));
+ ig2 = startGrid(getConfiguration("node2"));
+
+ info(">>>> reconnect client");
+
+ reconnectedLatch.await();
+
+ info(">>>> client reconnected");
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ info(">>>> activate new servers");
+
+ client.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkAllCaches();
+ }
+
+ public void testInActiveClientReconnectToActiveCluster() throws Exception {
+ CountDownLatch activateLatch = new CountDownLatch(1);
+
+ startNodes(activateLatch);
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ stopGrid(node1);
+ stopGrid(node2);
+
+ disconnectedLatch.await();
+
+ ig1 = startGrid(getConfiguration(node1));
+ ig2 = startGrid(getConfiguration(node2));
+
+ ig1.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+
+ checkDescriptors(ig1, Collections.<String>emptySet());
+ checkDescriptors(ig2, Collections.<String>emptySet());
+
+ activateLatch.countDown();
+
+ reconnectedLatch.await();
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkOnlySystemCaches();
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ Set<String> exp2 = Sets.newHashSet(ccfgDynamicName, ccfgDynamicWithFilterName);
+
+ checkDescriptors(ig1, exp2);
+ checkDescriptors(ig2, exp2);
+ checkDescriptors(client, exp2);
+ }
+
+ public void testInActiveClientReconnectToInActiveCluster() throws Exception {
+ startNodes(null);
+
+ IgniteEx ig1 = grid(node1);
+ IgniteEx ig2 = grid(node2);
+ IgniteEx client = grid(nodeClient);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ final CountDownLatch disconnectedLatch = new CountDownLatch(1);
+ final CountDownLatch reconnectedLatch = new CountDownLatch(1);
+
+ addDisconnectListener(disconnectedLatch, reconnectedLatch);
+
+ stopGrid(node1);
+ stopGrid(node2);
+
+ assertTrue(!client.active());
+
+ disconnectedLatch.await();
+
+ ig1 = startGrid(getConfiguration(node1));
+ ig2 = startGrid(getConfiguration(node2));
+
+ reconnectedLatch.await();
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!client.active());
+
+ client.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(client.active());
+
+ checkOnlySystemCaches();
+
+ client.createCache(ccfgDynamic);
+
+ client.createCache(ccfgDynamicWithFilter);
+
+ Set<String> exp2 = Sets.newHashSet(ccfgDynamicName, ccfgDynamicWithFilterName);
+
+ checkDescriptors(ig1, exp2);
+ checkDescriptors(ig2, exp2);
+ checkDescriptors(client, exp2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java
new file mode 100644
index 0000000..b50f026
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIOTest.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.ThreadLocalRandom;
+import junit.framework.TestCase;
+import org.apache.ignite.internal.util.GridUnsafe;
+
+/**
+ *
+ */
+public class TrackingPageIOTest extends TestCase {
+ /** Page size. */
+ public static final int PAGE_SIZE = 2048;
+
+ /** */
+ private final TrackingPageIO io = TrackingPageIO.VERSIONS.latest();
+
+ /**
+ *
+ */
+ public void testBasics() {
+ ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ io.markChanged(buf, 2, 0, -1, PAGE_SIZE);
+
+ assertTrue(io.wasChanged(buf, 2, 0, -1, PAGE_SIZE));
+
+ assertFalse(io.wasChanged(buf, 1, 0, -1, PAGE_SIZE));
+ assertFalse(io.wasChanged(buf, 3, 0, -1, PAGE_SIZE));
+ assertFalse(io.wasChanged(buf, 2, 1, 0, PAGE_SIZE));
+ }
+
+ /**
+ *
+ */
+ public void testMarkingRandomly() {
+ ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ int cntOfPageToTrack = io.countOfPageToTrack(PAGE_SIZE);
+
+ for (int i = 0; i < 1001; i++)
+ checkMarkingRandomly(buf, i, false);
+ }
+
+ /**
+ *
+ */
+ public void testZeroingRandomly() {
+ ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ for (int i = 0; i < 1001; i++)
+ checkMarkingRandomly(buf, i, true);
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param backupId Backup id.
+ */
+ private void checkMarkingRandomly(ByteBuffer buf, int backupId, boolean testZeroing) {
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ int track = io.countOfPageToTrack(PAGE_SIZE);
+
+ long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
+
+ long maxId = testZeroing ? basePageId + rand.nextInt(1, track) : basePageId + track;
+
+ assert basePageId >= 0;
+
+ PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
+
+ Map<Long, Boolean> map = new HashMap<>();
+
+ int cntOfChanged = 0;
+
+ try {
+ for (long i = basePageId; i < basePageId + track; i++) {
+ boolean changed = (i == basePageId || rand.nextDouble() < 0.5) && i < maxId;
+
+ map.put(i, changed);
+
+ if (changed) {
+ io.markChanged(buf, i, backupId, backupId - 1, PAGE_SIZE);
+
+ cntOfChanged++;
+ }
+
+ assertEquals(basePageId, PageIO.getPageId(buf));
+ assertEquals(cntOfChanged, io.countOfChangedPage(buf, backupId, PAGE_SIZE));
+ }
+
+ assertEquals(cntOfChanged, io.countOfChangedPage(buf, backupId, PAGE_SIZE));
+
+ for (Map.Entry<Long, Boolean> e : map.entrySet())
+ assertEquals(
+ e.getValue().booleanValue(),
+ io.wasChanged(buf, e.getKey(), backupId, backupId -1, PAGE_SIZE));
+ }
+ catch (Throwable e) {
+ System.out.println("snapshotId = " + backupId + ", basePageId = " + basePageId);
+ throw e;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFindNextChangedPage() throws Exception {
+ ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ for (int i = 0; i < 101; i++)
+ checkFindingRandomly(buf, i);
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param backupId Backup id.
+ */
+ private void checkFindingRandomly(ByteBuffer buf, int backupId) {
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ int track = io.countOfPageToTrack(PAGE_SIZE);
+
+ long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
+
+ long maxId = basePageId + rand.nextInt(1, track);
+
+ assert basePageId >= 0;
+
+ PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
+
+ try {
+ TreeSet<Long> setIdx = new TreeSet<>();
+
+ generateMarking(buf, track, basePageId, maxId, setIdx, backupId, backupId -1);
+
+ for (long pageId = basePageId; pageId < basePageId + track; pageId++) {
+ Long foundNextChangedPage = io.findNextChangedPage(buf, pageId, backupId, backupId - 1, PAGE_SIZE);
+
+ if (io.trackingPageFor(pageId, PAGE_SIZE) == pageId)
+ assertEquals((Long) pageId, foundNextChangedPage);
+
+ else if (setIdx.contains(pageId))
+ assertEquals((Long) pageId, foundNextChangedPage);
+
+ else {
+ NavigableSet<Long> tailSet = setIdx.tailSet(pageId, false);
+ Long next = tailSet.isEmpty() ? null : tailSet.first();
+
+ assertEquals(next, foundNextChangedPage);
+ }
+ }
+ }
+ catch (Throwable e) {
+ System.out.println("snapshotId = " + backupId + ", basePageId = " + basePageId);
+ throw e;
+ }
+ }
+
+ /**
+ *
+ */
+ public void testMerging() {
+ ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ int track = io.countOfPageToTrack(PAGE_SIZE);
+
+ long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
+
+ assert basePageId >= 0;
+
+ PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
+
+ TreeSet<Long> setIdx = new TreeSet<>();
+
+ for (int i = 0; i < 4; i++)
+ generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx, i, -1);
+
+ TreeSet<Long> setIdx2 = new TreeSet<>();
+
+ generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx2, 4, -1);
+
+ assertEquals(setIdx2.size(), io.countOfChangedPage(buf, 4, PAGE_SIZE));
+ assertEquals(setIdx.size(), io.countOfChangedPage(buf, 3, PAGE_SIZE));
+
+ for (long i = basePageId; i < basePageId + track; i++)
+ assertEquals("pageId = " + i, setIdx.contains(i), io.wasChanged(buf, i, 3, -1, PAGE_SIZE));
+
+ for (long i = basePageId; i < basePageId + track; i++)
+ assertEquals("pageId = " + i, setIdx2.contains(i), io.wasChanged(buf, i, 4, 3, PAGE_SIZE));
+
+ for (long i = basePageId; i < basePageId + track; i++)
+ assertFalse(io.wasChanged(buf, i, 5, 4, PAGE_SIZE));
+ }
+
+ /**
+ *
+ */
+ public void testMerging_MarksShouldBeDropForSuccessfulBackup() {
+ ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
+ buf.order(ByteOrder.nativeOrder());
+
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ int track = io.countOfPageToTrack(PAGE_SIZE);
+
+ long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
+
+ assert basePageId >= 0;
+
+ PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
+
+ TreeSet<Long> setIdx = new TreeSet<>();
+
+ for (int i = 0; i < 4; i++)
+ generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx, i, -1);
+
+ setIdx.clear();
+
+ generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx, 4, -1);
+
+ TreeSet<Long> setIdx2 = new TreeSet<>();
+
+ generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx2, 5, 3);
+
+ assertEquals(setIdx.size(), io.countOfChangedPage(buf, 4, PAGE_SIZE));
+ assertEquals(setIdx2.size(), io.countOfChangedPage(buf, 5, PAGE_SIZE));
+
+ for (long i = basePageId; i < basePageId + track; i++)
+ assertEquals("pageId = " + i, setIdx2.contains(i), io.wasChanged(buf, i, 5, 4, PAGE_SIZE));
+ }
+
+ private void generateMarking(
+ ByteBuffer buf,
+ int track,
+ long basePageId,
+ long maxPageId,
+ Set<Long> setIdx,
+ int backupId, int successfulBackupId
+ ) {
+ ThreadLocalRandom rand = ThreadLocalRandom.current();
+
+ for (long i = basePageId; i < basePageId + track; i++) {
+ boolean changed = (i == basePageId || rand.nextDouble() < 0.1) && i < maxPageId;
+
+ if (changed) {
+ io.markChanged(buf, i, backupId, successfulBackupId, PAGE_SIZE);
+
+ setIdx.add(i);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeFakeReuseSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeFakeReuseSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeFakeReuseSelfTest.java
index 83a07ab..cab54a4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeFakeReuseSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeFakeReuseSelfTest.java
@@ -20,8 +20,8 @@ package org.apache.ignite.internal.processors.database;
import java.util.concurrent.ConcurrentLinkedDeque;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
/**
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
index e5375f7..d12c92f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeReuseSelfTest.java
@@ -22,9 +22,9 @@ import java.util.Set;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
import static org.apache.ignite.internal.pagemem.PageIdUtils.effectivePageId;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
index a46242c..fadfc23 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/BPlusTreeSelfTest.java
@@ -41,15 +41,15 @@ import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.PageUtils;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.util.GridConcurrentHashSet;
import org.apache.ignite.internal.util.GridRandom;
import org.apache.ignite.internal.util.GridStripedLock;
@@ -65,7 +65,7 @@ import org.jsr166.ConcurrentHashMap8;
import org.jsr166.ConcurrentLinkedHashMap;
import static org.apache.ignite.internal.pagemem.PageIdUtils.effectivePageId;
-import static org.apache.ignite.internal.processors.cache.database.tree.BPlusTree.rnd;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.rnd;
import static org.apache.ignite.internal.util.IgniteTree.OperationType.NOOP;
import static org.apache.ignite.internal.util.IgniteTree.OperationType.PUT;
import static org.apache.ignite.internal.util.IgniteTree.OperationType.REMOVE;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
index 7ccddb7..d650ac1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/FreeListImplSelfTest.java
@@ -39,12 +39,12 @@ import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.CacheObjectContext;
import org.apache.ignite.internal.processors.cache.CacheObjectValueContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
-import org.apache.ignite.internal.processors.cache.database.evict.NoOpPageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeList;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.evict.NoOpPageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeList;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.plugin.extensions.communication.MessageReader;
import org.apache.ignite.plugin.extensions.communication.MessageWriter;
[25/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicyInitializationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicyInitializationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicyInitializationTest.java
deleted file mode 100644
index ec798ee..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicyInitializationTest.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.util.Collection;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.configuration.MemoryConfiguration.DFLT_MEM_PLC_DEFAULT_NAME;
-
-/**
- *
- */
-public class MemoryPolicyInitializationTest extends GridCommonAbstractTest {
- /** */
- private static final String CUSTOM_NON_DEFAULT_MEM_PLC_NAME = "custom_mem_plc";
-
- /** */
- private static final long USER_CUSTOM_MEM_PLC_SIZE = 10 * 1024 * 1024;
-
- /** */
- private static final long USER_DEFAULT_MEM_PLC_SIZE = 99 * 1024 * 1024;
-
- /** */
- private MemoryConfiguration memCfg;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
- cfg.setMemoryConfiguration(memCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- memCfg = null;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
- }
-
- /**
- * Verifies that expected memory policies are allocated when used doesn't provide any MemoryPolicyConfiguration.
- */
- public void testNoConfigProvided() throws Exception {
- memCfg = null;
-
- IgniteEx ignite = startGrid(0);
-
- Collection<MemoryPolicy> allMemPlcs = ignite.context().cache().context().database().memoryPolicies();
-
- assertTrue(allMemPlcs.size() == 2);
-
- verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
- }
-
- /**
- * Verifies that expected memory policies are allocated when used provides MemoryPolicyConfiguration
- * with non-default custom MemoryPolicy.
- */
- public void testCustomConfigNoDefault() throws Exception {
- prepareCustomNoDefaultConfig();
-
- IgniteEx ignite = startGrid(0);
-
- Collection<MemoryPolicy> allMemPlcs = ignite.context().cache().context().database().memoryPolicies();
-
- assertTrue(allMemPlcs.size() == 3);
-
- verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
-
- assertTrue("Custom non-default memory policy is not presented",
- isMemoryPolicyPresented(allMemPlcs, CUSTOM_NON_DEFAULT_MEM_PLC_NAME));
- }
-
- /**
- * User is allowed to configure memory policy with 'default' name,
- * in that case Ignite instance will use this user-defined memory policy as a default one.
- */
- public void testCustomConfigOverridesDefault() throws Exception {
- prepareCustomConfigWithOverridingDefault();
-
- IgniteEx ignite = startGrid(0);
-
- IgniteCacheDatabaseSharedManager dbMgr = ignite.context().cache().context().database();
-
- Collection<MemoryPolicy> allMemPlcs = dbMgr.memoryPolicies();
-
- assertTrue(allMemPlcs.size() == 2);
-
- verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
-
- MemoryPolicy dfltMemPlc = U.field(dbMgr, "dfltMemPlc");
-
- assertTrue(dfltMemPlc.config().getMaxSize() == USER_DEFAULT_MEM_PLC_SIZE);
- }
-
- /**
- * User is allowed to define fully custom memory policy and make it default by setting its name to memory config.
- *
- * At the same time user still can create a memory policy with name 'default'
- * which although won't be used as default.
- */
- public void testCustomConfigOverridesDefaultNameAndDeclaresDefault() throws Exception {
- prepareCustomConfigWithOverriddenDefaultName();
-
- IgniteEx ignite = startGrid(0);
-
- IgniteCacheDatabaseSharedManager dbMgr = ignite.context().cache().context().database();
-
- Collection<MemoryPolicy> allMemPlcs = dbMgr.memoryPolicies();
-
- assertTrue(allMemPlcs.size() == 3);
-
- verifyDefaultAndSystemMemoryPolicies(allMemPlcs);
-
- MemoryPolicy dfltMemPlc = U.field(dbMgr, "dfltMemPlc");
-
- assertTrue(dfltMemPlc.config().getMaxSize() == USER_CUSTOM_MEM_PLC_SIZE);
- }
-
- /**
- * Test for verification that caches with not specified memory policy name,
- * with specified default memory policy name and specified custom memory policy name
- * all started with correct memory policy.
- */
- public void testCachesOnOverriddenMemoryPolicy() throws Exception {
- prepareCustomConfigWithOverridingDefaultAndCustom();
-
- IgniteEx ignite = startGrid(0);
-
- CacheConfiguration cache1Cfg = new CacheConfiguration()
- .setName("cache1");
-
- IgniteCache cache1 = ignite.createCache(cache1Cfg);
-
- verifyCacheMemoryPolicy(cache1, DFLT_MEM_PLC_DEFAULT_NAME);
-
- CacheConfiguration cache2Cfg = new CacheConfiguration()
- .setName("cache2")
- .setMemoryPolicyName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
-
- IgniteCache cache2 = ignite.createCache(cache2Cfg);
-
- verifyCacheMemoryPolicy(cache2, CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
-
- CacheConfiguration cache3Cfg = new CacheConfiguration()
- .setName("cache3")
- .setMemoryPolicyName(DFLT_MEM_PLC_DEFAULT_NAME);
-
- IgniteCache cache3 = ignite.createCache(cache3Cfg);
-
- verifyCacheMemoryPolicy(cache3, DFLT_MEM_PLC_DEFAULT_NAME);
- }
-
- /**
- * Test for verification that caches with not specified memory policy name,
- * with specified default memory policy name and specified custom memory policy name
- * all started with correct memory policy.
- */
- public void testCachesOnUserDefinedDefaultMemoryPolicy() throws Exception {
- prepareCustomConfigWithOverriddenDefaultName();
-
- IgniteEx ignite = startGrid(0);
-
- CacheConfiguration cache1Cfg = new CacheConfiguration()
- .setName("cache1");
-
- IgniteCache cache1 = ignite.createCache(cache1Cfg);
-
- verifyCacheMemoryPolicy(cache1, CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
-
- CacheConfiguration cache2Cfg = new CacheConfiguration()
- .setName("cache2")
- .setMemoryPolicyName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
-
- IgniteCache cache2 = ignite.createCache(cache2Cfg);
-
- verifyCacheMemoryPolicy(cache2, CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
-
- CacheConfiguration cache3Cfg = new CacheConfiguration()
- .setName("cache3")
- .setMemoryPolicyName(DFLT_MEM_PLC_DEFAULT_NAME);
-
- IgniteCache cache3 = ignite.createCache(cache3Cfg);
-
- verifyCacheMemoryPolicy(cache3, DFLT_MEM_PLC_DEFAULT_NAME);
- }
-
- /**
- * @param cache Cache.
- * @param plcName Policy name.
- */
- private void verifyCacheMemoryPolicy(IgniteCache cache, String plcName) {
- GridCacheContext ctx = U.field(cache, "ctx");
-
- assertEquals(plcName, ctx.memoryPolicy().config().getName());
- }
-
- /**
- *
- */
- private void prepareCustomConfigWithOverriddenDefaultName() {
- memCfg = new MemoryConfiguration();
-
- memCfg.setDefaultMemoryPolicyName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME);
-
- memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
- .setName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME)
- .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
- .setMaxSize(USER_CUSTOM_MEM_PLC_SIZE),
-
- new MemoryPolicyConfiguration()
- .setName(DFLT_MEM_PLC_DEFAULT_NAME)
- .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
- .setMaxSize(USER_DEFAULT_MEM_PLC_SIZE)
- );
- }
-
-
- /**
- *
- */
- private void prepareCustomConfigWithOverridingDefault() {
- memCfg = new MemoryConfiguration();
-
- memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
- .setName(DFLT_MEM_PLC_DEFAULT_NAME)
- .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
- .setMaxSize(USER_DEFAULT_MEM_PLC_SIZE)
- );
- }
-
- /**
- *
- */
- private void prepareCustomConfigWithOverridingDefaultAndCustom() {
- memCfg = new MemoryConfiguration();
-
- memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
- .setName(DFLT_MEM_PLC_DEFAULT_NAME)
- .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
- .setMaxSize(USER_DEFAULT_MEM_PLC_SIZE),
-
- new MemoryPolicyConfiguration()
- .setName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME)
- .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
- .setMaxSize(USER_CUSTOM_MEM_PLC_SIZE)
- );
- }
-
- /**
- * @param allMemPlcs Collection of all memory policies.
- */
- private void verifyDefaultAndSystemMemoryPolicies(Collection<MemoryPolicy> allMemPlcs) {
- assertTrue("Default memory policy is not presented",
- isMemoryPolicyPresented(allMemPlcs, DFLT_MEM_PLC_DEFAULT_NAME));
-
- assertTrue("System memory policy is not presented",
- isMemoryPolicyPresented(allMemPlcs, IgniteCacheDatabaseSharedManager.SYSTEM_MEMORY_POLICY_NAME));
- }
-
- /**
- *
- */
- private void prepareCustomNoDefaultConfig() {
- memCfg = new MemoryConfiguration();
-
- memCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
- .setName(CUSTOM_NON_DEFAULT_MEM_PLC_NAME)
- .setInitialSize(USER_CUSTOM_MEM_PLC_SIZE)
- .setMaxSize(USER_CUSTOM_MEM_PLC_SIZE)
- );
- }
-
- /**
- * @param memPlcs Collection of memory policies.
- * @param nameToVerify Excepted name of memory policy.
- */
- private boolean isMemoryPolicyPresented(Collection<MemoryPolicy> memPlcs, String nameToVerify) {
- for (MemoryPolicy memPlc : memPlcs) {
- if (nameToVerify.equals(memPlc.config().getName()))
- return true;
- }
-
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIOTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIOTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIOTest.java
deleted file mode 100644
index e2767bb..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/database/tree/io/TrackingPageIOTest.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ThreadLocalRandom;
-import junit.framework.TestCase;
-import org.apache.ignite.internal.util.GridUnsafe;
-
-/**
- *
- */
-public class TrackingPageIOTest extends TestCase {
- /** Page size. */
- public static final int PAGE_SIZE = 2048;
-
- /** */
- private final TrackingPageIO io = TrackingPageIO.VERSIONS.latest();
-
- /**
- *
- */
- public void testBasics() {
- ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- io.markChanged(buf, 2, 0, -1, PAGE_SIZE);
-
- assertTrue(io.wasChanged(buf, 2, 0, -1, PAGE_SIZE));
-
- assertFalse(io.wasChanged(buf, 1, 0, -1, PAGE_SIZE));
- assertFalse(io.wasChanged(buf, 3, 0, -1, PAGE_SIZE));
- assertFalse(io.wasChanged(buf, 2, 1, 0, PAGE_SIZE));
- }
-
- /**
- *
- */
- public void testMarkingRandomly() {
- ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- int cntOfPageToTrack = io.countOfPageToTrack(PAGE_SIZE);
-
- for (int i = 0; i < 1001; i++)
- checkMarkingRandomly(buf, i, false);
- }
-
- /**
- *
- */
- public void testZeroingRandomly() {
- ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- for (int i = 0; i < 1001; i++)
- checkMarkingRandomly(buf, i, true);
- }
-
- /**
- * @param buf Buffer.
- * @param backupId Backup id.
- */
- private void checkMarkingRandomly(ByteBuffer buf, int backupId, boolean testZeroing) {
- ThreadLocalRandom rand = ThreadLocalRandom.current();
-
- int track = io.countOfPageToTrack(PAGE_SIZE);
-
- long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
-
- long maxId = testZeroing ? basePageId + rand.nextInt(1, track) : basePageId + track;
-
- assert basePageId >= 0;
-
- PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
-
- Map<Long, Boolean> map = new HashMap<>();
-
- int cntOfChanged = 0;
-
- try {
- for (long i = basePageId; i < basePageId + track; i++) {
- boolean changed = (i == basePageId || rand.nextDouble() < 0.5) && i < maxId;
-
- map.put(i, changed);
-
- if (changed) {
- io.markChanged(buf, i, backupId, backupId - 1, PAGE_SIZE);
-
- cntOfChanged++;
- }
-
- assertEquals(basePageId, PageIO.getPageId(buf));
- assertEquals(cntOfChanged, io.countOfChangedPage(buf, backupId, PAGE_SIZE));
- }
-
- assertEquals(cntOfChanged, io.countOfChangedPage(buf, backupId, PAGE_SIZE));
-
- for (Map.Entry<Long, Boolean> e : map.entrySet())
- assertEquals(
- e.getValue().booleanValue(),
- io.wasChanged(buf, e.getKey(), backupId, backupId -1, PAGE_SIZE));
- }
- catch (Throwable e) {
- System.out.println("snapshotId = " + backupId + ", basePageId = " + basePageId);
- throw e;
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testFindNextChangedPage() throws Exception {
- ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- for (int i = 0; i < 101; i++)
- checkFindingRandomly(buf, i);
- }
-
- /**
- * @param buf Buffer.
- * @param backupId Backup id.
- */
- private void checkFindingRandomly(ByteBuffer buf, int backupId) {
- ThreadLocalRandom rand = ThreadLocalRandom.current();
-
- int track = io.countOfPageToTrack(PAGE_SIZE);
-
- long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
-
- long maxId = basePageId + rand.nextInt(1, track);
-
- assert basePageId >= 0;
-
- PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
-
- try {
- TreeSet<Long> setIdx = new TreeSet<>();
-
- generateMarking(buf, track, basePageId, maxId, setIdx, backupId, backupId -1);
-
- for (long pageId = basePageId; pageId < basePageId + track; pageId++) {
- Long foundNextChangedPage = io.findNextChangedPage(buf, pageId, backupId, backupId - 1, PAGE_SIZE);
-
- if (io.trackingPageFor(pageId, PAGE_SIZE) == pageId)
- assertEquals((Long) pageId, foundNextChangedPage);
-
- else if (setIdx.contains(pageId))
- assertEquals((Long) pageId, foundNextChangedPage);
-
- else {
- NavigableSet<Long> tailSet = setIdx.tailSet(pageId, false);
- Long next = tailSet.isEmpty() ? null : tailSet.first();
-
- assertEquals(next, foundNextChangedPage);
- }
- }
- }
- catch (Throwable e) {
- System.out.println("snapshotId = " + backupId + ", basePageId = " + basePageId);
- throw e;
- }
- }
-
- /**
- *
- */
- public void testMerging() {
- ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- ThreadLocalRandom rand = ThreadLocalRandom.current();
-
- int track = io.countOfPageToTrack(PAGE_SIZE);
-
- long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
-
- assert basePageId >= 0;
-
- PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
-
- TreeSet<Long> setIdx = new TreeSet<>();
-
- for (int i = 0; i < 4; i++)
- generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx, i, -1);
-
- TreeSet<Long> setIdx2 = new TreeSet<>();
-
- generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx2, 4, -1);
-
- assertEquals(setIdx2.size(), io.countOfChangedPage(buf, 4, PAGE_SIZE));
- assertEquals(setIdx.size(), io.countOfChangedPage(buf, 3, PAGE_SIZE));
-
- for (long i = basePageId; i < basePageId + track; i++)
- assertEquals("pageId = " + i, setIdx.contains(i), io.wasChanged(buf, i, 3, -1, PAGE_SIZE));
-
- for (long i = basePageId; i < basePageId + track; i++)
- assertEquals("pageId = " + i, setIdx2.contains(i), io.wasChanged(buf, i, 4, 3, PAGE_SIZE));
-
- for (long i = basePageId; i < basePageId + track; i++)
- assertFalse(io.wasChanged(buf, i, 5, 4, PAGE_SIZE));
- }
-
- /**
- *
- */
- public void testMerging_MarksShouldBeDropForSuccessfulBackup() {
- ByteBuffer buf = ByteBuffer.allocateDirect(PAGE_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- ThreadLocalRandom rand = ThreadLocalRandom.current();
-
- int track = io.countOfPageToTrack(PAGE_SIZE);
-
- long basePageId = io.trackingPageFor(Math.max(rand.nextLong(Integer.MAX_VALUE - track), 0), PAGE_SIZE);
-
- assert basePageId >= 0;
-
- PageIO.setPageId(GridUnsafe.bufferAddress(buf), basePageId);
-
- TreeSet<Long> setIdx = new TreeSet<>();
-
- for (int i = 0; i < 4; i++)
- generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx, i, -1);
-
- setIdx.clear();
-
- generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx, 4, -1);
-
- TreeSet<Long> setIdx2 = new TreeSet<>();
-
- generateMarking(buf, track, basePageId, basePageId + rand.nextInt(1, track), setIdx2, 5, 3);
-
- assertEquals(setIdx.size(), io.countOfChangedPage(buf, 4, PAGE_SIZE));
- assertEquals(setIdx2.size(), io.countOfChangedPage(buf, 5, PAGE_SIZE));
-
- for (long i = basePageId; i < basePageId + track; i++)
- assertEquals("pageId = " + i, setIdx2.contains(i), io.wasChanged(buf, i, 5, 4, PAGE_SIZE));
- }
-
- private void generateMarking(
- ByteBuffer buf,
- int track,
- long basePageId,
- long maxPageId,
- Set<Long> setIdx,
- int backupId, int successfulBackupId
- ) {
- ThreadLocalRandom rand = ThreadLocalRandom.current();
-
- for (long i = basePageId; i < basePageId + track; i++) {
- boolean changed = (i == basePageId || rand.nextDouble() < 0.1) && i < maxPageId;
-
- if (changed) {
- io.markChanged(buf, i, backupId, successfulBackupId, PAGE_SIZE);
-
- setIdx.add(i);
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheRebalancingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheRebalancingTest.java
new file mode 100644
index 0000000..21551b6
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsAtomicCacheRebalancingTest.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+
+/**
+ *
+ */
+public class IgnitePdsAtomicCacheRebalancingTest extends IgnitePdsCacheRebalancingAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected CacheConfiguration cacheConfiguration(String cacheName) {
+ CacheConfiguration ccfg = new CacheConfiguration(cacheName);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setCacheMode(CacheMode.PARTITIONED);
+ ccfg.setBackups(1);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ return ccfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
new file mode 100644
index 0000000..a9428f8
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsCacheRebalancingAbstractTest.java
@@ -0,0 +1,523 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.PartitionLossPolicy;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+
+import static org.apache.ignite.testframework.GridTestUtils.runMultiThreadedAsync;
+
+/**
+ * Test for rebalancing and persistence integration.
+ */
+public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Cache name. */
+ private final String cacheName = "cache";
+
+ /** */
+ protected boolean explicitTx;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration ccfg1 = cacheConfiguration(cacheName);
+ ccfg1.setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
+ ccfg1.setBackups(1);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ CacheConfiguration ccfg2 = cacheConfiguration("indexed");
+ ccfg2.setBackups(1);
+ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ QueryEntity qryEntity = new QueryEntity(Integer.class.getName(), TestValue.class.getName());
+
+ LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+
+ fields.put("v1", Integer.class.getName());
+ fields.put("v2", Integer.class.getName());
+
+ qryEntity.setFields(fields);
+
+ QueryIndex qryIdx = new QueryIndex("v1", true);
+
+ qryEntity.setIndexes(Collections.singleton(qryIdx));
+
+ ccfg2.setQueryEntities(Collections.singleton(qryEntity));
+
+ cfg.setCacheConfiguration(ccfg1, ccfg2);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ memCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+ memCfg.setPageSize(1024);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setSwapFilePath("work/swap");
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setDiscoverySpi(
+ new TcpDiscoverySpi()
+ .setIpFinder(IP_FINDER)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 20 * 60 * 1000;
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @return Cache configuration.
+ */
+ protected abstract CacheConfiguration cacheConfiguration(String cacheName);
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * Test that outdated partitions on restarted nodes are correctly replaced with newer versions.
+ *
+ * @throws Exception If fails.
+ */
+ public void testRebalancingOnRestart() throws Exception {
+ Ignite ignite0 = startGrid(0);
+
+ startGrid(1);
+
+ IgniteEx ignite2 = startGrid(2);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache1 = ignite0.cache(cacheName);
+
+ for (int i = 0; i < 5000; i++)
+ cache1.put(i, i);
+
+ ignite2.close();
+
+ awaitPartitionMapExchange();
+
+ ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
+
+ assert cache1.lostPartitions().isEmpty();
+
+ for (int i = 0; i < 5000; i++)
+ cache1.put(i, i * 2);
+
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+
+ info(">>> Done puts...");
+
+ ignite2 = startGrid(2);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache3 = ignite2.cache(cacheName);
+
+ for (int i = 0; i < 100; i++)
+ assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
+ }
+
+ /**
+ * Test that outdated partitions on restarted nodes are correctly replaced with newer versions.
+ *
+ * @throws Exception If fails.
+ */
+ public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
+ IgniteEx ignite0 = startGrid(0);
+
+ IgniteEx ignite1 = startGrid(1);
+
+ IgniteEx ignite2 = startGrid(2);
+ IgniteEx ignite3 = startGrid(3);
+
+ ignite0.cache(cacheName).rebalance().get();
+ ignite1.cache(cacheName).rebalance().get();
+ ignite2.cache(cacheName).rebalance().get();
+ ignite3.cache(cacheName).rebalance().get();
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache1 = ignite0.cache(cacheName);
+
+ for (int i = 0; i < 1000; i++)
+ cache1.put(i, i);
+
+ ignite0.context().cache().context().database().waitForCheckpoint("test");
+ ignite1.context().cache().context().database().waitForCheckpoint("test");
+
+ info("++++++++++ After checkpoint");
+
+ ignite2.close();
+ ignite3.close();
+
+ awaitPartitionMapExchange();
+
+ ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
+
+ assert cache1.lostPartitions().isEmpty();
+
+ for (int i = 0; i < 1000; i++)
+ cache1.put(i, i * 2);
+
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>");
+
+ info(">>> Done puts...");
+
+ ignite2 = startGrid(2);
+ ignite3 = startGrid(3);
+
+ ignite2.cache(cacheName).rebalance().get();
+ ignite3.cache(cacheName).rebalance().get();
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache2 = ignite2.cache(cacheName);
+ IgniteCache<Integer, Integer> cache3 = ignite3.cache(cacheName);
+
+ for (int i = 0; i < 100; i++) {
+ assertEquals(String.valueOf(i), (Integer)(i * 2), cache2.get(i));
+ assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
+ }
+ }
+
+ /**
+ * Test that all data is correctly restored after non-graceful restart.
+ *
+ * @throws Exception If fails.
+ */
+ public void testDataCorrectnessAfterRestart() throws Exception {
+ IgniteEx ignite1 = (IgniteEx)G.start(getConfiguration("test1"));
+ IgniteEx ignite2 = (IgniteEx)G.start(getConfiguration("test2"));
+ IgniteEx ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
+ IgniteEx ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<Integer, Integer> cache1 = ignite1.cache(cacheName);
+
+ for (int i = 0; i < 100; i++)
+ cache1.put(i, i);
+
+ ignite1.close();
+ ignite2.close();
+ ignite3.close();
+ ignite4.close();
+
+ ignite1 = (IgniteEx)G.start(getConfiguration("test1"));
+ ignite2 = (IgniteEx)G.start(getConfiguration("test2"));
+ ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
+ ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
+
+ awaitPartitionMapExchange();
+
+ cache1 = ignite1.cache(cacheName);
+ IgniteCache<Integer, Integer> cache2 = ignite2.cache(cacheName);
+ IgniteCache<Integer, Integer> cache3 = ignite3.cache(cacheName);
+ IgniteCache<Integer, Integer> cache4 = ignite4.cache(cacheName);
+
+ for (int i = 0; i < 100; i++) {
+ assert cache1.get(i).equals(i);
+ assert cache2.get(i).equals(i);
+ assert cache3.get(i).equals(i);
+ assert cache4.get(i).equals(i);
+ }
+ }
+
+ /**
+ * Test that partitions are marked as lost when all owners leave cluster, but recover after nodes rejoin.
+ *
+ * @throws Exception If fails.
+ */
+ public void testPartitionLossAndRecover() throws Exception {
+ fail("IGNITE-5302");
+
+ Ignite ignite1 = startGrid(0);
+ Ignite ignite2 = startGrid(1);
+ Ignite ignite3 = startGrid(2);
+ Ignite ignite4 = startGrid(3);
+
+ awaitPartitionMapExchange();
+
+ IgniteCache<String, String> cache1 = ignite1.cache(cacheName);
+
+ final int offset = 10;
+
+ for (int i = 0; i < 100; i++)
+ cache1.put(String.valueOf(i), String.valueOf(i + offset));
+
+ ignite3.close();
+ ignite4.close();
+
+ awaitPartitionMapExchange();
+
+ assert !ignite1.cache(cacheName).lostPartitions().isEmpty();
+
+ ignite3 = startGrid(2);
+ ignite4 = startGrid(3);
+
+ ignite1.resetLostPartitions(Collections.singletonList(cacheName));
+
+ IgniteCache<String, String> cache2 = ignite2.cache(cacheName);
+ IgniteCache<String, String> cache3 = ignite3.cache(cacheName);
+ IgniteCache<String, String> cache4 = ignite4.cache(cacheName);
+
+ //Thread.sleep(5_000);
+
+ for (int i = 0; i < 100; i++) {
+ String key = String.valueOf(i);
+ String expected = String.valueOf(i + offset);
+
+ assertEquals(expected, cache1.get(key));
+ assertEquals(expected, cache2.get(key));
+ assertEquals(expected, cache3.get(key));
+ assertEquals(expected, cache4.get(key));
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testTopologyChangesWithConstantLoad() throws Exception {
+ final int entriesCnt = 10_000;
+ int maxNodesCount = 4;
+ int topChanges = 20;
+ final String cacheName = "indexed";
+
+ final AtomicBoolean stop = new AtomicBoolean();
+
+ final ConcurrentMap<Integer, TestValue> map = new ConcurrentHashMap<>();
+
+ Ignite ignite = startGrid(0);
+
+ IgniteCache<Integer, TestValue> cache = ignite.cache(cacheName);
+
+ for (int i = 0; i < entriesCnt; i++) {
+ cache.put(i, new TestValue(i, i));
+ map.put(i, new TestValue(i, i));
+ }
+
+ final AtomicInteger nodesCnt = new AtomicInteger();
+
+ IgniteInternalFuture fut = runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ while (true) {
+ if (stop.get())
+ return null;
+
+ int k = ThreadLocalRandom.current().nextInt(entriesCnt);
+ int v1 = ThreadLocalRandom.current().nextInt();
+ int v2 = ThreadLocalRandom.current().nextInt();
+
+ int n = nodesCnt.get();
+
+ if (n <= 0)
+ continue;
+
+ Ignite ignite;
+
+ try {
+ ignite = grid(ThreadLocalRandom.current().nextInt(n));
+ }
+ catch (Exception ignored) {
+ continue;
+ }
+
+ if (ignite == null)
+ continue;
+
+ Transaction tx = null;
+ boolean success = true;
+
+ if (explicitTx)
+ tx = ignite.transactions().txStart();
+
+ try {
+ ignite.cache(cacheName).put(k, new TestValue(v1, v2));
+ }
+ catch (Exception ignored) {
+ success = false;
+ }
+ finally {
+ if (tx != null) {
+ try {
+ tx.commit();
+ }
+ catch (Exception ignored) {
+ success = false;
+ }
+ }
+ }
+
+ if (success)
+ map.put(k, new TestValue(v1, v2));
+ }
+ }
+ }, 1, "load-runner");
+
+ for (int i = 0; i < topChanges; i++) {
+ U.sleep(3_000);
+
+ boolean add;
+
+ if (nodesCnt.get() <= maxNodesCount / 2)
+ add = true;
+ else if (nodesCnt.get() > maxNodesCount)
+ add = false;
+ else
+ add = ThreadLocalRandom.current().nextBoolean();
+
+ if (add)
+ startGrid(nodesCnt.incrementAndGet());
+ else
+ stopGrid(nodesCnt.getAndDecrement());
+
+ awaitPartitionMapExchange();
+
+ cache.rebalance().get();
+ }
+
+ stop.set(true);
+
+ fut.get();
+
+ awaitPartitionMapExchange();
+
+ for (Map.Entry<Integer, TestValue> entry : map.entrySet())
+ assertEquals(Integer.toString(entry.getKey()), entry.getValue(), cache.get(entry.getKey()));
+ }
+
+ /**
+ *
+ */
+ private static class TestValue implements Serializable {
+ /** V 1. */
+ private final int v1;
+ /** V 2. */
+ private final int v2;
+
+ /**
+ * @param v1 V 1.
+ * @param v2 V 2.
+ */
+ private TestValue(int v1, int v2) {
+ this.v1 = v1;
+ this.v2 = v2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ TestValue val = (TestValue)o;
+
+ return v1 == val.v1 && v2 == val.v2;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = v1;
+
+ res = 31 * res + v2;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "TestValue{" +
+ "v1=" + v1 +
+ ", v2=" + v2 +
+ '}';
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java
new file mode 100644
index 0000000..17dad10
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsClientNearCachePutGetTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class IgnitePdsClientNearCachePutGetTest extends IgniteDbClientNearCachePutGetTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
new file mode 100644
index 0000000..3721031
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Map;
+import java.util.Random;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
+ /** */
+ private static final int GRID_CNT = 4;
+
+ /** */
+ private static final int ENTRIES_COUNT = 10_000;
+
+ /** */
+ public static final String CACHE_NAME = "cache1";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setMaxSize(400 * 1024 * 1024);
+ memPlcCfg.setInitialSize(400 * 1024 * 1024);
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ CacheConfiguration ccfg1 = new CacheConfiguration();
+
+ ccfg1.setName(CACHE_NAME);
+ ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg1.setAffinity(new RendezvousAffinityFunction(false, 128));
+ ccfg1.setBackups(2);
+
+ cfg.setCacheConfiguration(ccfg1);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_1000_500_1_1() throws Exception {
+ checkRebalancingDuringLoad(1000, 500, 1, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_8000_500_1_1() throws Exception {
+ checkRebalancingDuringLoad(8000, 500, 1, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_1000_20000_1_1() throws Exception {
+ checkRebalancingDuringLoad(1000, 20000, 1, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_8000_8000_1_1() throws Exception {
+ checkRebalancingDuringLoad(8000, 8000, 1, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_1000_500_8_1() throws Exception {
+ checkRebalancingDuringLoad(1000, 500, 8, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_8000_500_8_1() throws Exception {
+ checkRebalancingDuringLoad(8000, 500, 8, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_1000_20000_8_1() throws Exception {
+ checkRebalancingDuringLoad(1000, 20000, 8, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_8000_8000_8_1() throws Exception {
+ checkRebalancingDuringLoad(8000, 8000, 8, 1);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_1000_500_8_16() throws Exception {
+ checkRebalancingDuringLoad(1000, 500, 8, 16);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_8000_500_8_16() throws Exception {
+ checkRebalancingDuringLoad(8000, 500, 8, 16);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_1000_20000_8_16() throws Exception {
+ checkRebalancingDuringLoad(1000, 20000, 8, 16);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalancingDuringLoad_8000_8000_8_16() throws Exception {
+ checkRebalancingDuringLoad(8000, 8000, 8, 16);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ private void checkRebalancingDuringLoad(
+ int restartDelay,
+ int checkpointDelay,
+ int threads,
+ final int batch
+ ) throws Exception {
+
+ startGrids(GRID_CNT);
+
+ final Ignite load = ignite(0);
+
+ try (IgniteDataStreamer<Object, Object> s = load.dataStreamer(CACHE_NAME)) {
+ s.allowOverwrite(true);
+
+ for (int i = 0; i < ENTRIES_COUNT; i++)
+ s.addData(i, i);
+ }
+
+ final AtomicBoolean done = new AtomicBoolean(false);
+
+ IgniteInternalFuture<?> busyFut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+ /** {@inheritDoc} */
+ @Override public Object call() throws Exception {
+ IgniteCache<Object, Object> cache = load.cache(CACHE_NAME);
+ Random rnd = ThreadLocalRandom.current();
+
+ while (!done.get()) {
+ Map<Integer, Integer> map = new TreeMap<>();
+
+ for (int i = 0; i < batch; i++)
+ map.put(rnd.nextInt(ENTRIES_COUNT), rnd.nextInt());
+
+ cache.putAll(map);
+ }
+
+ return null;
+ }
+ }, threads, "updater");
+
+ long end = System.currentTimeMillis() + 90_000;
+
+ Random rnd = ThreadLocalRandom.current();
+
+ while (System.currentTimeMillis() < end) {
+ int idx = rnd.nextInt(GRID_CNT - 1) + 1;
+
+ stopGrid(idx);
+
+ U.sleep(restartDelay);
+
+ startGrid(idx);
+
+ U.sleep(restartDelay);
+ }
+
+ done.set(true);
+
+ busyFut.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java
new file mode 100644
index 0000000..ba55c09
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsDynamicCacheTest.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class IgnitePdsDynamicCacheTest extends IgniteDbDynamicCacheSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(200 * 1024 * 1024);
+ memPlcCfg.setMaxSize(200 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ if ("client".equals(gridName))
+ cfg.setClientMode(true);
+
+ return cfg;
+ }
+
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC, "true");
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRestartAndCreate() throws Exception {
+ startGrids(3);
+
+ Ignite ignite = ignite(0);
+
+ CacheConfiguration ccfg1 = new CacheConfiguration();
+
+ ccfg1.setName("cache1");
+ ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg1.setRebalanceMode(CacheRebalanceMode.NONE);
+ ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ CacheConfiguration ccfg2 = new CacheConfiguration();
+
+ ccfg2.setName("cache2");
+ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg2.setRebalanceMode(CacheRebalanceMode.NONE);
+ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ ignite.createCache(ccfg1);
+ ignite.createCache(ccfg2);
+
+ int iterations = 20;
+
+ long stopTime = U.currentTimeMillis() + 20_000;
+
+ for (int k = 0; k < iterations && U.currentTimeMillis() < stopTime; k++) {
+ log.info("Iteration: " + k);
+
+ stopAllGrids();
+
+ startGrids(3);
+
+ ignite = ignite(0);
+
+ ignite.getOrCreateCache(ccfg1);
+
+ ignite.getOrCreateCache(ccfg2);
+
+ ignite.destroyCache(ccfg2.getName());
+
+ ignite.getOrCreateCache(ccfg2);
+
+ ignite.destroyCache(ccfg1.getName());
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsMultiNodePutGetRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsMultiNodePutGetRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsMultiNodePutGetRestartTest.java
new file mode 100644
index 0000000..21ea626
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsMultiNodePutGetRestartTest.java
@@ -0,0 +1,252 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.List;
+import java.util.UUID;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsMultiNodePutGetRestartTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int GRID_CNT = 3;
+
+ /** */
+ private final File allocPath;
+
+ /**
+ * Default constructor.
+ */
+ public IgnitePdsMultiNodePutGetRestartTest() {
+ String home = U.getIgniteHome();
+
+ allocPath = new File(home, "work/db/" + UUID.randomUUID());
+
+ allocPath.mkdirs();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+ memCfg.setMemoryPolicies(memPlcCfg);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setIndexedTypes(Integer.class, DbValue.class);
+
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration bCfg = new BinaryConfiguration();
+
+ bCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(bCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPutGetSimple() throws Exception {
+ info(">>> Will use path: " + allocPath);
+
+ startGrids(GRID_CNT);
+
+ try {
+ IgniteEx ig = grid(0);
+
+ checkPutGetSql(ig, true);
+ }
+ finally {
+ stopAllGrids();
+ }
+
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+
+ startGrids(GRID_CNT);
+
+ try {
+ IgniteEx ig = grid(0);
+
+ checkPutGetSql(ig, false);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ig Ig.
+ * @param write Write.
+ */
+ private void checkPutGetSql(IgniteEx ig, boolean write) {
+ IgniteCache<Integer, DbValue> cache = ig.cache(null);
+
+ if (write) {
+ try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(null)) {
+ for (int i = 0; i < 10_000; i++)
+ streamer.addData(i, new DbValue(i, "value-" + i, i));
+ }
+ }
+
+ List<List<?>> res = cache.query(new SqlFieldsQuery("select ival from dbvalue where ival < ? order by ival asc")
+ .setArgs(10_000)).getAll();
+
+ assertEquals(10_000, res.size());
+
+ for (int i = 0; i < 10_000; i++) {
+ assertEquals(1, res.get(i).size());
+ assertEquals(i, res.get(i).get(0));
+ }
+
+ assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
+ assertEquals(5000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 10000"))
+ .getAll().size());
+
+ for (int i = 0; i < 10_000; i++)
+ assertEquals(new DbValue(i, "value-" + i, i), cache.get(i));
+ }
+
+ /**
+ *
+ */
+ private static class DbValue implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String sVal;
+
+ /** */
+ @QuerySqlField
+ private long lVal;
+
+ /**
+ * @param iVal Integer value.
+ * @param sVal String value.
+ * @param lVal Long value.
+ */
+ public DbValue(int iVal, String sVal, long lVal) {
+ this.iVal = iVal;
+ this.sVal = sVal;
+ this.lVal = lVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DbValue dbVal = (DbValue)o;
+
+ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DbValue.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPageSizesTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPageSizesTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPageSizesTest.java
new file mode 100644
index 0000000..5014399
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsPageSizesTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsPageSizesTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private final String cacheName = "cache";
+
+ /** */
+ private int pageSize;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ memCfg.setPageSize(pageSize);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setCacheConfiguration(
+ new CacheConfiguration(cacheName)
+ .setAffinity(new RendezvousAffinityFunction(false, 32))
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageSize_1k() throws Exception {
+ checkPageSize(1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageSize_2k() throws Exception {
+ checkPageSize(2 * 1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageSize_4k() throws Exception {
+ checkPageSize(4 * 1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageSize_8k() throws Exception {
+ checkPageSize(8 * 1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageSize_16k() throws Exception {
+ checkPageSize(16 * 1024);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ private void checkPageSize(int pageSize) throws Exception {
+ this.pageSize = pageSize;
+
+ IgniteEx ignite = startGrid(0);
+
+ try {
+ final IgniteCache<Object, Object> cache = ignite.cache(cacheName);
+ final long endTime = System.currentTimeMillis() + 60_000;
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ Random rnd = ThreadLocalRandom.current();
+
+ while (System.currentTimeMillis() < endTime) {
+ for (int i = 0; i < 500; i++)
+ cache.put(rnd.nextInt(100_000), rnd.nextInt());
+ }
+
+ return null;
+ }
+ }, 16, "runner");
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
new file mode 100644
index 0000000..4981095
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsRecoveryAfterFileCorruptionTest.java
@@ -0,0 +1,361 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.util.Collection;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStore;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstractTest {
+ /** Ip finder. */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Total pages. */
+ private static final int totalPages = 1024;
+
+ /** Cache name. */
+ private final String cacheName = "cache";
+
+ /** Policy name. */
+ private final String policyName = "dfltMemPlc";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration ccfg = new CacheConfiguration(cacheName);
+ ccfg.setAffinity(new RendezvousAffinityFunction(true, 1));
+
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName(policyName);
+ memPlcCfg.setInitialSize(1024 * 1024 * 1024);
+ memPlcCfg.setMaxSize(1024 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName(policyName);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setCheckpointingFrequency(500)
+ .setAlwaysWriteFullPages(true)
+ );
+
+ cfg.setDiscoverySpi(
+ new TcpDiscoverySpi()
+ .setIpFinder(ipFinder)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageRecoveryAfterFileCorruption() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
+
+ // Put for create data store and init meta page.
+ cache.put(1, 1);
+
+ GridCacheSharedContext sharedCtx = ig.context().cache().context();
+
+ GridCacheDatabaseSharedManager psMgr = (GridCacheDatabaseSharedManager)sharedCtx.database();
+
+ FilePageStoreManager pageStore = (FilePageStoreManager)sharedCtx.pageStore();
+
+ U.sleep(1_000);
+
+ // Disable integrated checkpoint thread.
+ psMgr.enableCheckpoints(false).get();
+
+ PageMemory mem = sharedCtx.database().memoryPolicy(policyName).pageMemory();
+
+ int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
+
+ FullPageId[] pages = new FullPageId[totalPages];
+
+ for (int i = 0; i < totalPages; i++)
+ pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+
+ generateWal(
+ (PageMemoryImpl)mem,
+ sharedCtx.pageStore(),
+ sharedCtx.wal(),
+ cacheId,
+ pages
+ );
+
+ eraseDataFromDisk(pageStore, cacheId, pages[0]);
+
+ stopAllGrids();
+
+ ig = startGrid(0);
+
+ checkRestore(ig, pages);
+ }
+
+ /**
+ * @param pageStore Page store.
+ * @param cacheId Cache id.
+ * @param page Page.
+ */
+ private void eraseDataFromDisk(
+ FilePageStoreManager pageStore,
+ int cacheId,
+ FullPageId page
+ ) throws IgniteCheckedException, IOException {
+ PageStore store = pageStore.getStore(
+ cacheId,
+ PageIdUtils.partId(page.pageId())
+ );
+
+ FilePageStore filePageStore = (FilePageStore)store;
+
+ FileChannel ch = U.field(filePageStore, "ch");
+
+ long size = ch.size();
+
+ ch.write(ByteBuffer.allocate((int)size - FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE);
+
+ ch.force(false);
+ }
+
+ /**
+ * @param ig Ig.
+ * @param pages Pages.
+ */
+ private void checkRestore(IgniteEx ig, FullPageId[] pages) throws IgniteCheckedException {
+ GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ PageMemory mem = shared.database().memoryPolicy(null).pageMemory();
+
+ for (FullPageId fullId : pages) {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ long pageAddr = mem.readLock(fullId.cacheId(), fullId.pageId(), page);
+
+ for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
+ assertEquals(j + (int)fullId.pageId(), PageUtils.getInt(pageAddr, j));
+
+ mem.readUnlock(fullId.cacheId(), fullId.pageId(), page);
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ }
+
+ /**
+ * @param mem Mem.
+ * @param storeMgr Store manager.
+ * @param wal Wal.
+ * @param cacheId Cache id.
+ * @param pages Pages.
+ */
+ private void generateWal(
+ final PageMemoryImpl mem,
+ final IgnitePageStoreManager storeMgr,
+ final IgniteWriteAheadLogManager wal,
+ final int cacheId, FullPageId[] pages
+ ) throws Exception {
+ // Mark the start position.
+ CheckpointRecord cpRec = new CheckpointRecord(null, false);
+
+ WALPointer start = wal.log(cpRec);
+
+ wal.fsync(start);
+
+ for (int i = 0; i < totalPages; i++) {
+ FullPageId fullId = pages[i];
+
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ PageIO.setPageId(pageAddr, fullId.pageId());
+
+ try {
+ for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
+ PageUtils.putInt(pageAddr, j, j + (int)fullId.pageId());
+ }
+ finally {
+ mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+
+ Collection<FullPageId> pageIds = mem.beginCheckpoint();
+
+ info("Acquired pages for checkpoint: " + pageIds.size());
+
+ try {
+ ByteBuffer tmpBuf = ByteBuffer.allocate(mem.pageSize());
+
+ tmpBuf.order(ByteOrder.nativeOrder());
+
+ long begin = System.currentTimeMillis();
+
+ long cp = 0;
+
+ long write = 0;
+
+ for (int i = 0; i < totalPages; i++) {
+ FullPageId fullId = pages[i];
+
+ if (pageIds.contains(fullId)) {
+ long cpStart = System.nanoTime();
+
+ Integer tag = mem.getForCheckpoint(fullId, tmpBuf, null);
+
+ if (tag == null)
+ continue;
+
+ long cpEnd = System.nanoTime();
+
+ cp += cpEnd - cpStart;
+ tmpBuf.rewind();
+
+ for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
+ assertEquals(j + (int)fullId.pageId(), tmpBuf.getInt(j));
+
+ tmpBuf.rewind();
+
+ long writeStart = System.nanoTime();
+
+ storeMgr.write(cacheId, fullId.pageId(), tmpBuf, tag);
+
+ long writeEnd = System.nanoTime();
+
+ write += writeEnd - writeStart;
+
+ tmpBuf.rewind();
+ }
+ }
+
+ long syncStart = System.currentTimeMillis();
+
+ storeMgr.sync(cacheId, 0);
+
+ long end = System.currentTimeMillis();
+
+ info("Written pages in " + (end - begin) + "ms, copy took " + (cp / 1_000_000) + "ms, " +
+ "write took " + (write / 1_000_000) + "ms, sync took " + (end - syncStart) + "ms");
+ }
+ finally {
+ info("Finishing checkpoint...");
+
+ mem.finishCheckpoint();
+
+ info("Finished checkpoint");
+ }
+
+ wal.fsync(wal.log(new CheckpointRecord(null, false)));
+
+ for (FullPageId fullId : pages) {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" +
+ fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page));
+
+ assertFalse("Page is dirty after the last checkpoint: [cacheId=" +
+ fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
[42/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
deleted file mode 100644
index fe1a0d0..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusIO.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.lang.IgniteInClosure;
-
-/**
- * Abstract IO routines for B+Tree pages.
- */
-public abstract class BPlusIO<L> extends PageIO {
- /** */
- private static final int CNT_OFF = COMMON_HEADER_END;
-
- /** */
- private static final int FORWARD_OFF = CNT_OFF + 2;
-
- /** */
- private static final int REMOVE_ID_OFF = FORWARD_OFF + 8;
-
- /** */
- protected static final int ITEMS_OFF = REMOVE_ID_OFF + 8;
-
- /** */
- private final boolean canGetRow;
-
- /** */
- private final boolean leaf;
-
- /** All the items must be of fixed size. */
- protected final int itemSize;
-
- /**
- * @param type Page type.
- * @param ver Page format version.
- * @param leaf If this is a leaf IO.
- * @param canGetRow If we can get full row from this page.
- */
- protected BPlusIO(int type, int ver, boolean leaf, boolean canGetRow, int itemSize) {
- super(type, ver);
-
- assert itemSize > 0 : itemSize;
- assert canGetRow || !leaf: "leaf page always must be able to get full row";
-
- this.leaf = leaf;
- this.canGetRow = canGetRow;
- this.itemSize = itemSize;
- }
-
- /**
- * @return Item size in bytes.
- */
- public final int getItemSize() {
- return itemSize;
- }
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setCount(pageAddr, 0);
- setForward(pageAddr, 0);
- setRemoveId(pageAddr, 0);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Forward page ID.
- */
- public final long getForward(long pageAddr) {
- return PageUtils.getLong(pageAddr, FORWARD_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageId Forward page ID.
- */
- public final void setForward(long pageAddr, long pageId) {
- PageUtils.putLong(pageAddr, FORWARD_OFF, pageId);
-
- assert getForward(pageAddr) == pageId;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Remove ID.
- */
- public final long getRemoveId(long pageAddr) {
- return PageUtils.getLong(pageAddr, REMOVE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param rmvId Remove ID.
- */
- public final void setRemoveId(long pageAddr, long rmvId) {
- PageUtils.putLong(pageAddr, REMOVE_ID_OFF, rmvId);
-
- assert getRemoveId(pageAddr) == rmvId;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Items count in the page.
- */
- public final int getCount(long pageAddr) {
- int cnt = PageUtils.getShort(pageAddr, CNT_OFF) & 0xFFFF;
-
- assert cnt >= 0: cnt;
-
- return cnt;
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Count.
- */
- public final void setCount(long pageAddr, int cnt) {
- assert cnt >= 0: cnt;
-
- PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
-
- assert getCount(pageAddr) == cnt;
- }
-
- /**
- * @return {@code true} If we can get the full row from this page using
- * method {@link BPlusTree#getRow(BPlusIO, long, int)}.
- * Must always be {@code true} for leaf pages.
- */
- public final boolean canGetRow() {
- return canGetRow;
- }
-
- /**
- * @return {@code true} if it is a leaf page.
- */
- public final boolean isLeaf() {
- return leaf;
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- * @return Max items count.
- */
- public abstract int getMaxCount(long pageAddr, int pageSize);
-
- /**
- * Store the needed info about the row in the page. Leaf and inner pages can store different info.
- *
- * @param pageAddr Page address.
- * @param idx Index.
- * @param row Lookup or full row.
- * @param rowBytes Row bytes.
- * @param needRowBytes If we need stored row bytes.
- * @return Stored row bytes.
- * @throws IgniteCheckedException If failed.
- */
- public final byte[] store(long pageAddr, int idx, L row, byte[] rowBytes, boolean needRowBytes)
- throws IgniteCheckedException {
- int off = offset(idx);
-
- if (rowBytes == null) {
- storeByOffset(pageAddr, off, row);
-
- if (needRowBytes)
- rowBytes = PageUtils.getBytes(pageAddr, off, getItemSize());
- }
- else
- putBytes(pageAddr, off, rowBytes);
-
- return rowBytes;
- }
-
- /**
- * @param idx Index of element.
- * @return Offset from byte buffer begin in bytes.
- */
- public abstract int offset(int idx);
-
- /**
- * Store the needed info about the row in the page. Leaf and inner pages can store different info.
- *
- * @param pageAddr Page address.
- * @param off Offset in bytes.
- * @param row Lookup or full row.
- * @throws IgniteCheckedException If failed.
- */
- public abstract void storeByOffset(long pageAddr, int off, L row) throws IgniteCheckedException;
-
- /**
- * Store row info from the given source.
- *
- * @param dstPageAddr Destination page address.
- * @param dstIdx Destination index.
- * @param srcIo Source IO.
- * @param srcPageAddr Source page address.
- * @param srcIdx Source index.
- * @throws IgniteCheckedException If failed.
- */
- public abstract void store(long dstPageAddr, int dstIdx, BPlusIO<L> srcIo, long srcPageAddr, int srcIdx)
- throws IgniteCheckedException;
-
- /**
- * Get lookup row.
- *
- * @param tree Tree.
- * @param pageAddr Page address.
- * @param idx Index.
- * @return Lookup row.
- * @throws IgniteCheckedException If failed.
- */
- public abstract L getLookupRow(BPlusTree<L, ?> tree, long pageAddr, int idx) throws IgniteCheckedException;
-
- /**
- * Copy items from source page to destination page.
- * Both pages must be of the same type and the same version.
- *
- * @param srcPageAddr Source page address.
- * @param dstPageAddr Destination page address.
- * @param srcIdx Source begin index.
- * @param dstIdx Destination begin index.
- * @param cnt Items count.
- * @param cpLeft Copy leftmost link (makes sense only for inner pages).
- * @throws IgniteCheckedException If failed.
- */
- public abstract void copyItems(long srcPageAddr, long dstPageAddr, int srcIdx, int dstIdx, int cnt, boolean cpLeft)
- throws IgniteCheckedException;
-
- // Methods for B+Tree logic.
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @param row Row to insert.
- * @param rowBytes Row bytes.
- * @param rightId Page ID which will be to the right child for the inserted item.
- * @param needRowBytes If we need stored row bytes.
- * @return Row bytes.
- * @throws IgniteCheckedException If failed.
- */
- public byte[] insert(long pageAddr, int idx, L row, byte[] rowBytes, long rightId, boolean needRowBytes)
- throws IgniteCheckedException {
- int cnt = getCount(pageAddr);
-
- // Move right all the greater elements to make a free slot for a new row link.
- copyItems(pageAddr, pageAddr, idx, idx + 1, cnt - idx, false);
-
- setCount(pageAddr, cnt + 1);
-
- return store(pageAddr, idx, row, rowBytes, needRowBytes);
- }
-
- /**
- * @param pageAddr Splitting page address.
- * @param fwdId Forward page ID.
- * @param fwdPageAddr Forward page address.
- * @param mid Bisection index.
- * @param cnt Initial elements count in the page being split.
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void splitForwardPage(
- long pageAddr,
- long fwdId,
- long fwdPageAddr,
- int mid,
- int cnt,
- int pageSize
- ) throws IgniteCheckedException {
- initNewPage(fwdPageAddr, fwdId, pageSize);
-
- cnt -= mid;
-
- copyItems(pageAddr, fwdPageAddr, mid, 0, cnt, true);
-
- setCount(fwdPageAddr, cnt);
- setForward(fwdPageAddr, getForward(pageAddr));
-
- // Copy remove ID to make sure that if inner remove touched this page, then retry
- // will happen even for newly allocated forward page.
- setRemoveId(fwdPageAddr, getRemoveId(pageAddr));
- }
-
- /**
- * @param pageAddr Page address.
- * @param mid Bisection index.
- * @param fwdId New forward page ID.
- */
- public void splitExistingPage(long pageAddr, int mid, long fwdId) {
- setCount(pageAddr, mid);
- setForward(pageAddr, fwdId);
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @param cnt Count.
- * @throws IgniteCheckedException If failed.
- */
- public void remove(long pageAddr, int idx, int cnt) throws IgniteCheckedException {
- cnt--;
-
- copyItems(pageAddr, pageAddr, idx + 1, idx, cnt - idx, false);
- setCount(pageAddr, cnt);
- }
-
- /**
- * @param prntIo Parent IO.
- * @param prntPageAddr Parent page address.
- * @param prntIdx Split key index in parent.
- * @param leftPageAddr Left page address.
- * @param rightPageAddr Right page address.
- * @param emptyBranch We are merging an empty branch.
- * @param pageSize Page size.
- * @return {@code false} If we were not able to merge.
- * @throws IgniteCheckedException If failed.
- */
- public boolean merge(
- BPlusIO<L> prntIo,
- long prntPageAddr,
- int prntIdx,
- long leftPageAddr,
- long rightPageAddr,
- boolean emptyBranch,
- int pageSize
- ) throws IgniteCheckedException {
- int prntCnt = prntIo.getCount(prntPageAddr);
- int leftCnt = getCount(leftPageAddr);
- int rightCnt = getCount(rightPageAddr);
-
- int newCnt = leftCnt + rightCnt;
-
- // Need to move down split key in inner pages. For empty branch merge parent key will be just dropped.
- if (!isLeaf() && !emptyBranch)
- newCnt++;
-
- if (newCnt > getMaxCount(leftPageAddr, pageSize)) {
- assert !emptyBranch;
-
- return false;
- }
-
- setCount(leftPageAddr, newCnt);
-
- // Move down split key in inner pages.
- if (!isLeaf() && !emptyBranch) {
- assert prntIdx >= 0 && prntIdx < prntCnt: prntIdx; // It must be adjusted already.
-
- // We can be sure that we have enough free space to store split key here,
- // because we've done remove already and did not release child locks.
- store(leftPageAddr, leftCnt, prntIo, prntPageAddr, prntIdx);
-
- leftCnt++;
- }
-
- copyItems(rightPageAddr, leftPageAddr, 0, leftCnt, rightCnt, !emptyBranch);
- setForward(leftPageAddr, getForward(rightPageAddr));
-
- long rmvId = getRemoveId(rightPageAddr);
-
- // Need to have maximum remove ID.
- if (rmvId > getRemoveId(leftPageAddr))
- setRemoveId(leftPageAddr, rmvId);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- * @param pos Position in page.
- * @param bytes Bytes.
- */
- private static void putBytes(long pageAddr, int pos, byte[] bytes) {
- PageUtils.putBytes(pageAddr, pos, bytes);
- }
-
- /**
- * @param pageAddr Page address.
- * @param c Closure.
- */
- public void visit(long pageAddr, IgniteInClosure<L> c) {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
deleted file mode 100644
index 60fd24c..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusInnerIO.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-
-/**
- * Abstract IO routines for B+Tree inner pages.
- */
-public abstract class BPlusInnerIO<L> extends BPlusIO<L> {
- /** */
- private static final int SHIFT_LEFT = ITEMS_OFF;
-
- /** */
- private static final int SHIFT_LINK = SHIFT_LEFT + 8;
-
- /** */
- private final int SHIFT_RIGHT = SHIFT_LINK + itemSize;
-
- /**
- * @param type Page type.
- * @param ver Page format version.
- * @param canGetRow If we can get full row from this page.
- * @param itemSize Single item size on page.
- */
- protected BPlusInnerIO(int type, int ver, boolean canGetRow, int itemSize) {
- super(type, ver, false, canGetRow, itemSize);
- }
-
- /** {@inheritDoc} */
- @Override public int getMaxCount(long pageAddr, int pageSize) {
- // The structure of the page is the following:
- // |ITEMS_OFF|w|A|x|B|y|C|z|
- // where capital letters are data items, lowercase letters are 8 byte page references.
- return (pageSize - ITEMS_OFF - 8) / (getItemSize() + 8);
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @return Page ID.
- */
- public final long getLeft(long pageAddr, int idx) {
- return PageUtils.getLong(pageAddr, (offset0(idx, SHIFT_LEFT)));
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @param pageId Page ID.
- */
- public final void setLeft(long pageAddr, int idx, long pageId) {
- PageUtils.putLong(pageAddr, offset0(idx, SHIFT_LEFT), pageId);
-
- assert pageId == getLeft(pageAddr, idx);
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @return Page ID.
- */
- public final long getRight(long pageAddr, int idx) {
- return PageUtils.getLong(pageAddr, offset0(idx, SHIFT_RIGHT));
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @param pageId Page ID.
- */
- private void setRight(long pageAddr, int idx, long pageId) {
- PageUtils.putLong(pageAddr, offset0(idx, SHIFT_RIGHT), pageId);
-
- assert pageId == getRight(pageAddr, idx);
- }
-
- /** {@inheritDoc} */
- @Override public final void copyItems(long srcPageAddr, long dstPageAddr, int srcIdx, int dstIdx, int cnt,
- boolean cpLeft) throws IgniteCheckedException {
- assert srcIdx != dstIdx || srcPageAddr != dstPageAddr;
-
- cnt *= getItemSize() + 8; // From items to bytes.
-
- if (dstIdx > srcIdx) {
- PageHandler.copyMemory(srcPageAddr, dstPageAddr, offset(srcIdx), offset(dstIdx), cnt);
-
- if (cpLeft)
- PageUtils.putLong(dstPageAddr, offset0(dstIdx, SHIFT_LEFT), PageUtils.getLong(srcPageAddr, (offset0(srcIdx, SHIFT_LEFT))));
- }
- else {
- if (cpLeft)
- PageUtils.putLong(dstPageAddr, offset0(dstIdx, SHIFT_LEFT), PageUtils.getLong(srcPageAddr, (offset0(srcIdx, SHIFT_LEFT))));
-
- PageHandler.copyMemory(srcPageAddr, dstPageAddr, offset(srcIdx), offset(dstIdx), cnt);
- }
- }
-
- /**
- * @param idx Index of element.
- * @param shift It can be either link itself or left or right page ID.
- * @return Offset from byte buffer begin in bytes.
- */
- private int offset0(int idx, int shift) {
- return shift + (8 + getItemSize()) * idx;
- }
-
- /** {@inheritDoc} */
- @Override public final int offset(int idx) {
- return offset0(idx, SHIFT_LINK);
- }
-
- // Methods for B+Tree logic.
-
- /** {@inheritDoc} */
- @Override public byte[] insert(
- long pageAddr,
- int idx,
- L row,
- byte[] rowBytes,
- long rightId,
- boolean needRowBytes
- ) throws IgniteCheckedException {
- rowBytes = super.insert(pageAddr, idx, row, rowBytes, rightId, needRowBytes);
-
- // Setup reference to the right page on split.
- setRight(pageAddr, idx, rightId);
-
- return rowBytes;
- }
-
- /**
- * @param newRootPageAddr New root page address.
- * @param newRootId New root ID.
- * @param leftChildId Left child ID.
- * @param row Moved up row.
- * @param rowBytes Bytes.
- * @param rightChildId Right child ID.
- * @param pageSize Page size.
- * @param needRowBytes If we need row bytes back.
- * @return Row bytes.
- * @throws IgniteCheckedException If failed.
- */
- public byte[] initNewRoot(
- long newRootPageAddr,
- long newRootId,
- long leftChildId,
- L row,
- byte[] rowBytes,
- long rightChildId,
- int pageSize,
- boolean needRowBytes
- ) throws IgniteCheckedException {
- initNewPage(newRootPageAddr, newRootId, pageSize);
-
- setCount(newRootPageAddr, 1);
- setLeft(newRootPageAddr, 0, leftChildId);
- rowBytes = store(newRootPageAddr, 0, row, rowBytes, needRowBytes);
- setRight(newRootPageAddr, 0, rightChildId);
-
- return rowBytes;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
deleted file mode 100644
index f6011b3..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusLeafIO.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-
-/**
- * Abstract IO routines for B+Tree leaf pages.
- */
-public abstract class BPlusLeafIO<L> extends BPlusIO<L> {
- /**
- * @param type Page type.
- * @param ver Page format version.
- * @param itemSize Single item size on page.
- */
- protected BPlusLeafIO(int type, int ver, int itemSize) {
- super(type, ver, true, true, itemSize);
- }
-
- /** {@inheritDoc} */
- @Override public int getMaxCount(long pageAddr, int pageSize) {
- return (pageSize - ITEMS_OFF) / getItemSize();
- }
-
- /** {@inheritDoc} */
- @Override public final void copyItems(long srcPageAddr, long dstPageAddr, int srcIdx, int dstIdx, int cnt,
- boolean cpLeft) throws IgniteCheckedException {
- assert srcIdx != dstIdx || srcPageAddr != dstPageAddr;
-
- PageHandler.copyMemory(srcPageAddr, dstPageAddr, offset(srcIdx), offset(dstIdx),
- cnt * getItemSize());
- }
-
- /** {@inheritDoc} */
- @Override public final int offset(int idx) {
- assert idx >= 0: idx;
-
- return ITEMS_OFF + idx * getItemSize();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
deleted file mode 100644
index 6755820..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/BPlusMetaIO.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.pagemem.PageUtils;
-
-/**
- * IO routines for B+Tree meta pages.
- */
-public class BPlusMetaIO extends PageIO {
- /** */
- public static final IOVersions<BPlusMetaIO> VERSIONS = new IOVersions<>(
- new BPlusMetaIO(1), new BPlusMetaIO(2)
- );
-
- /** */
- private static final int LVLS_OFF = COMMON_HEADER_END;
-
- /** */
- private final int refsOff;
-
- /** */
- private final int inlineSizeOff;
-
- /**
- * @param ver Page format version.
- */
- private BPlusMetaIO(int ver) {
- super(T_BPLUS_META, ver);
-
- switch (ver) {
- case 1:
- inlineSizeOff = -1;
- refsOff = LVLS_OFF + 1;
- break;
-
- case 2:
- inlineSizeOff = LVLS_OFF + 1;
- refsOff = inlineSizeOff + 2;
- break;
-
- default:
- throw new IgniteException("invalid IO version: " + ver);
- }
- }
-
- /**
- * @param pageAdrr Page address.
- * @param rootId Root page ID.
- * @param pageSize Page size.
- */
- public void initRoot(long pageAdrr, long rootId, int pageSize) {
- setLevelsCount(pageAdrr, 1, pageSize);
- setFirstPageId(pageAdrr, 0, rootId);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Number of levels in this tree.
- */
- public int getLevelsCount(long pageAddr) {
- return PageUtils.getByte(pageAddr, LVLS_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- * @return Max levels possible for this page size.
- */
- private int getMaxLevels(long pageAddr, int pageSize) {
- return (pageSize - refsOff) / 8;
- }
-
- /**
- * @param pageAddr Page address.
- * @param lvls Number of levels in this tree.
- * @param pageSize Page size.
- */
- private void setLevelsCount(long pageAddr, int lvls, int pageSize) {
- assert lvls >= 0 && lvls <= getMaxLevels(pageAddr, pageSize) : lvls;
-
- PageUtils.putByte(pageAddr, LVLS_OFF, (byte)lvls);
-
- assert getLevelsCount(pageAddr) == lvls;
- }
-
- /**
- * @param lvl Level.
- * @return Offset for page reference.
- */
- private int offset(int lvl) {
- return lvl * 8 + refsOff;
- }
-
- /**
- * @param pageAddr Page address.
- * @param lvl Level.
- * @return First page ID at that level.
- */
- public long getFirstPageId(long pageAddr, int lvl) {
- return PageUtils.getLong(pageAddr, offset(lvl));
- }
-
- /**
- * @param pageAddr Page address.
- * @param lvl Level.
- * @param pageId Page ID.
- */
- private void setFirstPageId(long pageAddr, int lvl, long pageId) {
- assert lvl >= 0 && lvl < getLevelsCount(pageAddr) : lvl;
-
- PageUtils.putLong(pageAddr, offset(lvl), pageId);
-
- assert getFirstPageId(pageAddr, lvl) == pageId;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Root level.
- */
- public int getRootLevel(long pageAddr) {
- int lvls = getLevelsCount(pageAddr); // The highest level page is root.
-
- assert lvls > 0 : lvls;
-
- return lvls - 1;
- }
-
- /**
- * @param pageAddr Page address.
- * @param rootPageId New root page ID.
- * @param pageSize Page size.
- */
- public void addRoot(long pageAddr, long rootPageId, int pageSize) {
- int lvl = getLevelsCount(pageAddr);
-
- setLevelsCount(pageAddr, lvl + 1, pageSize);
- setFirstPageId(pageAddr, lvl, rootPageId);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- */
- public void cutRoot(long pageAddr, int pageSize) {
- int lvl = getRootLevel(pageAddr);
-
- setLevelsCount(pageAddr, lvl, pageSize); // Decrease tree height.
- }
-
- /**
- * @param pageAddr Page address.
- * @param size Offset size.
- */
- public void setInlineSize(long pageAddr, int size) {
- if (getVersion() > 1)
- PageUtils.putShort(pageAddr, inlineSizeOff, (short)size);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public int getInlineSize(long pageAddr) {
- return getVersion() > 1 ? PageUtils.getShort(pageAddr, inlineSizeOff) : 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/CacheVersionIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/CacheVersionIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/CacheVersionIO.java
deleted file mode 100644
index 7c4537e..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/CacheVersionIO.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-
-/**
- * Utility to read and write {@link GridCacheVersion} instances.
- */
-public class CacheVersionIO {
- /** */
- private static final byte NULL_PROTO_VER = 0;
-
- /** */
- private static final byte MAX_PROTO_VER = 1;
-
- /** */
- private static final int NULL_SIZE = 1;
-
- /** Serialized size in bytes. */
- private static final int SIZE_V1 = 17;
-
- /**
- * @param ver Version.
- * @param allowNull Is {@code null} version allowed.
- * @return Serialized size in bytes.
- */
- public static int size(GridCacheVersion ver, boolean allowNull) {
- if (ver == null) {
- if (allowNull)
- return NULL_SIZE;
-
- throw new IllegalStateException("Cache version is null");
- }
-
- return SIZE_V1;
- }
-
- /**
- * @param buf Byte buffer.
- * @param ver Version to write.
- * @param allowNull Is {@code null} version allowed.
- */
- public static void write(ByteBuffer buf, GridCacheVersion ver, boolean allowNull) {
- if (ver == null) {
- if (allowNull)
- buf.put(NULL_PROTO_VER);
- else
- throw new IllegalStateException("Cache version is null");
- }
- else {
- byte protoVer = 1; // Version of serialization protocol.
-
- buf.put(protoVer);
- buf.putInt(ver.topologyVersion());
- buf.putInt(ver.nodeOrderAndDrIdRaw());
- buf.putLong(ver.order());
- }
- }
-
- /**
- * @param addr Write address.
- * @param ver Version to write.
- * @param allowNull Is {@code null} version allowed.
- */
- public static void write(long addr, GridCacheVersion ver, boolean allowNull) {
- if (ver == null) {
- if (allowNull)
- PageUtils.putByte(addr, 0, NULL_PROTO_VER);
- else
- throw new IllegalStateException("Cache version is null");
- }
- else {
- byte protoVer = 1; // Version of serialization protocol.
-
- PageUtils.putByte(addr, 0, protoVer);
- PageUtils.putInt(addr, 1, ver.topologyVersion());
- PageUtils.putInt(addr, 5, ver.nodeOrderAndDrIdRaw());
- PageUtils.putLong(addr, 9, ver.order());
- }
- }
-
- /**
- * @param protoVer Serialization protocol version.
- * @param allowNull Is {@code null} version allowed.
- * @return Protocol version.
- * @throws IgniteCheckedException if failed.
- */
- private static byte checkProtocolVersion(byte protoVer, boolean allowNull) throws IgniteCheckedException {
- if (protoVer >= NULL_PROTO_VER && protoVer <= MAX_PROTO_VER) {
- if (protoVer == NULL_PROTO_VER && !allowNull)
- throw new IllegalStateException("Cache version is null.");
-
- return protoVer;
- }
-
- throw new IgniteCheckedException("Unsupported protocol version: " + protoVer);
- }
-
- /**
- * Gets needed buffer size to read the whole version instance.
- * Does not change buffer position.
- *
- * @param buf Buffer.
- * @param allowNull Is {@code null} version allowed.
- * @return Size of serialized version.
- * @throws IgniteCheckedException If failed.
- */
- public static int readSize(ByteBuffer buf, boolean allowNull) throws IgniteCheckedException {
- byte protoVer = checkProtocolVersion(buf.get(buf.position()), allowNull);
-
- switch (protoVer) {
- case NULL_PROTO_VER:
- return NULL_SIZE;
-
- case 1:
- return SIZE_V1;
-
- default:
- throw new IllegalStateException();
- }
- }
-
- /**
- * Reads GridCacheVersion instance from the given buffer. Moves buffer's position by the number of used
- * bytes.
- *
- * @param buf Byte buffer.
- * @param allowNull Is {@code null} version allowed.
- * @return Version.
- * @throws IgniteCheckedException If failed.
- */
- public static GridCacheVersion read(ByteBuffer buf, boolean allowNull) throws IgniteCheckedException {
- byte protoVer = checkProtocolVersion(buf.get(), allowNull);
-
- if (protoVer == NULL_PROTO_VER)
- return null;
-
- int topVer = buf.getInt();
- int nodeOrderDrId = buf.getInt();
- long order = buf.getLong();
-
- return new GridCacheVersion(topVer, nodeOrderDrId, order);
- }
-
- /**
- * Reads GridCacheVersion instance from the given address.
- *
- * @param pageAddr Page address.
- * @param allowNull Is {@code null} version allowed.
- * @return Version.
- * @throws IgniteCheckedException If failed.
- */
- public static GridCacheVersion read(long pageAddr, boolean allowNull) throws IgniteCheckedException {
- byte protoVer = checkProtocolVersion(PageUtils.getByte(pageAddr, 0), allowNull);
-
- if (protoVer == NULL_PROTO_VER)
- return null;
-
- int topVer = PageUtils.getInt(pageAddr, 1);
- int nodeOrderDrId = PageUtils.getInt(pageAddr, 5);
- long order = PageUtils.getLong(pageAddr, 9);
-
- return new GridCacheVersion(topVer, nodeOrderDrId, order);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
deleted file mode 100644
index 3ed69e1..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPageIO.java
+++ /dev/null
@@ -1,1450 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Data pages IO.
- */
-public class DataPageIO extends PageIO {
- /** */
- public static final IOVersions<DataPageIO> VERSIONS = new IOVersions<>(
- new DataPageIO(1)
- );
-
- /** */
- private static final int SHOW_ITEM = 0b0001;
-
- /** */
- private static final int SHOW_PAYLOAD_LEN = 0b0010;
-
- /** */
- private static final int SHOW_LINK = 0b0100;
-
- /** */
- private static final int FREE_LIST_PAGE_ID_OFF = COMMON_HEADER_END;
-
- /** */
- private static final int FREE_SPACE_OFF = FREE_LIST_PAGE_ID_OFF + 8;
-
- /** */
- private static final int DIRECT_CNT_OFF = FREE_SPACE_OFF + 2;
-
- /** */
- private static final int INDIRECT_CNT_OFF = DIRECT_CNT_OFF + 1;
-
- /** */
- private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
-
- /** */
- private static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
-
- /** */
- private static final int ITEM_SIZE = 2;
-
- /** */
- private static final int PAYLOAD_LEN_SIZE = 2;
-
- /** */
- private static final int LINK_SIZE = 8;
-
- /** */
- private static final int FRAGMENTED_FLAG = 0b10000000_00000000;
-
- /** */
- public static final int MIN_DATA_PAGE_OVERHEAD = ITEMS_OFF + ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
-
- /**
- * @param ver Page format version.
- */
- protected DataPageIO(int ver) {
- super(T_DATA, ver);
- }
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setEmptyPage(pageAddr, pageSize);
- setFreeListPageId(pageAddr, 0L);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- */
- private void setEmptyPage(long pageAddr, int pageSize) {
- setDirectCount(pageAddr, 0);
- setIndirectCount(pageAddr, 0);
- setFirstEntryOffset(pageAddr, pageSize, pageSize);
- setRealFreeSpace(pageAddr, pageSize - ITEMS_OFF, pageSize);
- }
-
- /**
- * @param pageAddr Page address.
- * @param freeListPageId Free list page ID.
- */
- public void setFreeListPageId(long pageAddr, long freeListPageId) {
- PageUtils.putLong(pageAddr, FREE_LIST_PAGE_ID_OFF, freeListPageId);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Free list page ID.
- */
- public long getFreeListPageId(long pageAddr) {
- return PageUtils.getLong(pageAddr, FREE_LIST_PAGE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param show What elements of data page entry to show in the result.
- * @return Data page entry size.
- */
- private int getPageEntrySize(long pageAddr, int dataOff, int show) {
- int payloadLen = PageUtils.getShort(pageAddr, dataOff) & 0xFFFF;
-
- if ((payloadLen & FRAGMENTED_FLAG) != 0)
- payloadLen &= ~FRAGMENTED_FLAG; // We are fragmented and have a link.
- else
- show &= ~SHOW_LINK; // We are not fragmented, never have a link.
-
- return getPageEntrySize(payloadLen, show);
- }
-
- /**
- * @param payloadLen Length of the payload, may be a full data row or a row fragment length.
- * @param show What elements of data page entry to show in the result.
- * @return Data page entry size.
- */
- private int getPageEntrySize(int payloadLen, int show) {
- assert payloadLen > 0 : payloadLen;
-
- int res = payloadLen;
-
- if ((show & SHOW_LINK) != 0)
- res += LINK_SIZE;
-
- if ((show & SHOW_ITEM) != 0)
- res += ITEM_SIZE;
-
- if ((show & SHOW_PAYLOAD_LEN) != 0)
- res += PAYLOAD_LEN_SIZE;
-
- return res;
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Entry data offset.
- * @param pageSize Page size.
- */
- private void setFirstEntryOffset(long pageAddr, int dataOff, int pageSize) {
- assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff <= pageSize : dataOff;
-
- PageUtils.putShort(pageAddr, FIRST_ENTRY_OFF, (short)dataOff);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Entry data offset.
- */
- private int getFirstEntryOffset(long pageAddr) {
- return PageUtils.getShort(pageAddr, FIRST_ENTRY_OFF) & 0xFFFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @param freeSpace Free space.
- * @param pageSize Page size.
- */
- private void setRealFreeSpace(long pageAddr, int freeSpace, int pageSize) {
- assert freeSpace == actualFreeSpace(pageAddr, pageSize) : freeSpace + " != " + actualFreeSpace(pageAddr, pageSize);
-
- PageUtils.putShort(pageAddr, FREE_SPACE_OFF, (short)freeSpace);
- }
-
- /**
- * Free space refers to a "max row size (without any data page specific overhead) which is
- * guaranteed to fit into this data page".
- *
- * @param pageAddr Page address.
- * @return Free space.
- */
- public int getFreeSpace(long pageAddr) {
- if (getFreeItemSlots(pageAddr) == 0)
- return 0;
-
- int freeSpace = getRealFreeSpace(pageAddr);
-
- // We reserve size here because of getFreeSpace() method semantics (see method javadoc).
- // It means that we must be able to accommodate a row of size which is equal to getFreeSpace(),
- // plus we will have data page overhead: header of the page as well as item, payload length and
- // possibly a link to the next row fragment.
- freeSpace -= ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
-
- return freeSpace < 0 ? 0 : freeSpace;
- }
-
- /**
- * @param pageAddr Page address.
- * @return {@code true} If there is no useful data in this page.
- */
- public boolean isEmpty(long pageAddr) {
- return getDirectCount(pageAddr) == 0;
- }
-
- /**
- * Equivalent for {@link #actualFreeSpace(long, int)} but reads saved value.
- *
- * @param pageAddr Page address.
- * @return Free space.
- */
- private int getRealFreeSpace(long pageAddr) {
- return PageUtils.getShort(pageAddr, FREE_SPACE_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Direct count.
- */
- private void setDirectCount(long pageAddr, int cnt) {
- assert checkCount(cnt): cnt;
-
- PageUtils.putByte(pageAddr, DIRECT_CNT_OFF, (byte)cnt);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Direct count.
- */
- private int getDirectCount(long pageAddr) {
- return PageUtils.getByte(pageAddr, DIRECT_CNT_OFF) & 0xFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @param c Closure.
- * @param <T> Closure return type.
- * @return Collection of closure results for all items in page.
- * @throws IgniteCheckedException In case of error in closure body.
- */
- public <T> List<T> forAllItems(long pageAddr, CC<T> c) throws IgniteCheckedException {
- long pageId = getPageId(pageAddr);
-
- int cnt = getDirectCount(pageAddr);
-
- List<T> res = new ArrayList<>(cnt);
-
- for (int i = 0; i < cnt; i++) {
- long link = PageIdUtils.link(pageId, i);
-
- res.add(c.apply(link));
- }
-
- return res;
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Indirect count.
- */
- private void setIndirectCount(long pageAddr, int cnt) {
- assert checkCount(cnt): cnt;
-
- PageUtils.putByte(pageAddr, INDIRECT_CNT_OFF, (byte)cnt);
- }
-
- /**
- * @param idx Index.
- * @return {@code true} If the index is valid.
- */
- private boolean checkIndex(int idx) {
- return idx >= 0 && idx < 0xFF;
- }
-
- /**
- * @param cnt Counter value.
- * @return {@code true} If the counter fits 1 byte.
- */
- private boolean checkCount(int cnt) {
- return cnt >= 0 && cnt <= 0xFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Indirect count.
- */
- private int getIndirectCount(long pageAddr) {
- return PageUtils.getByte(pageAddr, INDIRECT_CNT_OFF) & 0xFF;
- }
-
- /**
- * @param pageAddr Page address.
- * @return Number of free entry slots.
- */
- private int getFreeItemSlots(long pageAddr) {
- return 0xFF - getDirectCount(pageAddr);
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return Found index of indirect item.
- */
- private int findIndirectItemIndex(long pageAddr, int itemId, int directCnt, int indirectCnt) {
- int low = directCnt;
- int high = directCnt + indirectCnt - 1;
-
- while (low <= high) {
- int mid = (low + high) >>> 1;
-
- int cmp = Integer.compare(itemId(getItem(pageAddr, mid)), itemId);
-
- if (cmp < 0)
- low = mid + 1;
- else if (cmp > 0)
- high = mid - 1;
- else
- return mid; // found
- }
-
- throw new IllegalStateException("Item not found: " + itemId);
- }
-
- /**
- * @param pageAddr Page address.
- * @param pageSize Page size.
- * @return String representation.
- */
- private String printPageLayout(long pageAddr, int pageSize) {
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
- int free = getRealFreeSpace(pageAddr);
-
- boolean valid = directCnt >= indirectCnt;
-
- SB b = new SB();
-
- b.appendHex(PageIO.getPageId(pageAddr)).a(" [");
-
- int entriesSize = 0;
-
- for (int i = 0; i < directCnt; i++) {
- if (i != 0)
- b.a(", ");
-
- short item = getItem(pageAddr, i);
-
- if (item < ITEMS_OFF || item >= pageSize)
- valid = false;
-
- entriesSize += getPageEntrySize(pageAddr, item, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- b.a(item);
- }
-
- b.a("][");
-
- Collection<Integer> set = new HashSet<>();
-
- for (int i = directCnt; i < directCnt + indirectCnt; i++) {
- if (i != directCnt)
- b.a(", ");
-
- short item = getItem(pageAddr, i);
-
- int itemId = itemId(item);
- int directIdx = directItemIndex(item);
-
- if (!set.add(directIdx) || !set.add(itemId))
- valid = false;
-
- assert indirectItem(itemId, directIdx) == item;
-
- if (itemId < directCnt || directIdx < 0 || directIdx >= directCnt)
- valid = false;
-
- if (i > directCnt && itemId(getItem(pageAddr, i - 1)) >= itemId)
- valid = false;
-
-
- b.a(itemId).a('^').a(directIdx);
- }
-
- b.a("][free=").a(free);
-
- int actualFree = pageSize - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE);
-
- if (free != actualFree) {
- b.a(", actualFree=").a(actualFree);
-
- valid = false;
- }
- else
- b.a("]");
-
- assert valid : b.toString();
-
- return b.toString();
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param pageSize Page size.
- * @return Data entry offset in bytes.
- */
- private int getDataOffset(long pageAddr, int itemId, int pageSize) {
- assert checkIndex(itemId): itemId;
-
- int directCnt = getDirectCount(pageAddr);
-
- assert directCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", page=" + printPageLayout(pageAddr, pageSize);
-
- if (itemId >= directCnt) { // Need to do indirect lookup.
- int indirectCnt = getIndirectCount(pageAddr);
-
- // Must have indirect items here.
- assert indirectCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", indirectCnt=" + indirectCnt +
- ", page=" + printPageLayout(pageAddr, pageSize);
-
- int indirectItemIdx = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
-
- assert indirectItemIdx >= directCnt : indirectItemIdx + " " + directCnt;
- assert indirectItemIdx < directCnt + indirectCnt: indirectItemIdx + " " + directCnt + " " + indirectCnt;
-
- itemId = directItemIndex(getItem(pageAddr, indirectItemIdx));
-
- assert itemId >= 0 && itemId < directCnt: itemId + " " + directCnt + " " + indirectCnt; // Direct item.
- }
-
- return directItemToOffset(getItem(pageAddr, itemId));
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Points to the entry start.
- * @return Link to the next entry fragment or 0 if no fragments left or if entry is not fragmented.
- */
- private long getNextFragmentLink(long pageAddr, int dataOff) {
- assert isFragmented(pageAddr, dataOff);
-
- return PageUtils.getLong(pageAddr, dataOff + PAYLOAD_LEN_SIZE);
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @return {@code true} If the data row is fragmented across multiple pages.
- */
- private boolean isFragmented(long pageAddr, int dataOff) {
- return (PageUtils.getShort(pageAddr, dataOff) & FRAGMENTED_FLAG) != 0;
- }
-
- /**
- * Sets position to start of actual fragment data and limit to it's end.
- *
- * @param pageAddr Page address.
- * @param itemId Item to position on.
- * @param pageSize Page size.
- * @return {@link DataPagePayload} object.
- */
- public DataPagePayload readPayload(final long pageAddr, final int itemId, final int pageSize) {
- int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-
- boolean fragmented = isFragmented(pageAddr, dataOff);
- long nextLink = fragmented ? getNextFragmentLink(pageAddr, dataOff) : 0;
- int payloadSize = getPageEntrySize(pageAddr, dataOff, 0);
-
- return new DataPagePayload(dataOff + PAYLOAD_LEN_SIZE + (fragmented ? LINK_SIZE : 0),
- payloadSize,
- nextLink);
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Item index.
- * @return Item.
- */
- private short getItem(long pageAddr, int idx) {
- return PageUtils.getShort(pageAddr, itemOffset(idx));
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Item index.
- * @param item Item.
- */
- private void setItem(long pageAddr, int idx, short item) {
- PageUtils.putShort(pageAddr, itemOffset(idx), item);
- }
-
- /**
- * @param idx Index of the item.
- * @return Offset in buffer.
- */
- private int itemOffset(int idx) {
- assert checkIndex(idx): idx;
-
- return ITEMS_OFF + idx * ITEM_SIZE;
- }
-
- /**
- * @param directItem Direct item.
- * @return Offset of an entry payload inside of the page.
- */
- private int directItemToOffset(short directItem) {
- return directItem & 0xFFFF;
- }
-
- /**
- * @param dataOff Data offset.
- * @return Direct item.
- */
- private short directItemFromOffset(int dataOff) {
- assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff < Short.MAX_VALUE: dataOff;
-
- return (short)dataOff;
- }
-
- /**
- * @param indirectItem Indirect item.
- * @return Index of corresponding direct item.
- */
- private int directItemIndex(short indirectItem) {
- return indirectItem & 0xFF;
- }
-
- /**
- * @param indirectItem Indirect item.
- * @return Fixed item ID (the index used for referencing an entry from the outside).
- */
- private int itemId(short indirectItem) {
- return (indirectItem & 0xFFFF) >>> 8;
- }
-
- /**
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param directItemIdx Index of corresponding direct item.
- * @return Indirect item.
- */
- private short indirectItem(int itemId, int directItemIdx) {
- assert checkIndex(itemId): itemId;
- assert checkIndex(directItemIdx): directItemIdx;
-
- return (short)((itemId << 8) | directItemIdx);
- }
-
- /**
- * Move the last direct item to the free slot and reference it with indirect item on the same place.
- *
- * @param pageAddr Page address.
- * @param freeDirectIdx Free slot.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return {@code true} If the last direct item already had corresponding indirect item.
- */
- private boolean moveLastItem(long pageAddr, int freeDirectIdx, int directCnt, int indirectCnt) {
- int lastIndirectId = findIndirectIndexForLastDirect(pageAddr, directCnt, indirectCnt);
-
- int lastItemId = directCnt - 1;
-
- assert lastItemId != freeDirectIdx;
-
- short indirectItem = indirectItem(lastItemId, freeDirectIdx);
-
- assert itemId(indirectItem) == lastItemId && directItemIndex(indirectItem) == freeDirectIdx;
-
- setItem(pageAddr, freeDirectIdx, getItem(pageAddr, lastItemId));
- setItem(pageAddr, lastItemId, indirectItem);
-
- assert getItem(pageAddr, lastItemId) == indirectItem;
-
- if (lastIndirectId != -1) { // Fix pointer to direct item.
- setItem(pageAddr, lastIndirectId, indirectItem(itemId(getItem(pageAddr, lastIndirectId)), freeDirectIdx));
-
- return true;
- }
-
- return false;
- }
-
- /**
- * @param pageAddr Page address.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return Index of indirect item for the last direct item.
- */
- private int findIndirectIndexForLastDirect(long pageAddr, int directCnt, int indirectCnt) {
- int lastDirectId = directCnt - 1;
-
- for (int i = directCnt, end = directCnt + indirectCnt; i < end; i++) {
- short item = getItem(pageAddr, i);
-
- if (directItemIndex(item) == lastDirectId)
- return i;
- }
-
- return -1;
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Item ID.
- * @param pageSize Page size.
- * @param payload Row data.
- * @param row Row.
- * @param rowSize Row size.
- * @return {@code True} if entry is not fragmented.
- * @throws IgniteCheckedException If failed.
- */
- public boolean updateRow(
- final long pageAddr,
- int itemId,
- int pageSize,
- @Nullable byte[] payload,
- @Nullable CacheDataRow row,
- final int rowSize) throws IgniteCheckedException {
- assert checkIndex(itemId) : itemId;
- assert row != null ^ payload != null;
-
- final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
-
- if (isFragmented(pageAddr, dataOff))
- return false;
-
- if (row != null)
- writeRowData(pageAddr, dataOff, rowSize, row, false);
- else
- writeRowData(pageAddr, dataOff, payload);
-
- return true;
- }
-
- /**
- * @param pageAddr Page address.
- * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
- * @param pageSize Page size.
- * @return Next link for fragmented entries or {@code 0} if none.
- * @throws IgniteCheckedException If failed.
- */
- public long removeRow(long pageAddr, int itemId, int pageSize) throws IgniteCheckedException {
- assert checkIndex(itemId) : itemId;
-
- final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
- final long nextLink = isFragmented(pageAddr, dataOff) ? getNextFragmentLink(pageAddr, dataOff) : 0;
-
- // Record original counts to calculate delta in free space in the end of remove.
- final int directCnt = getDirectCount(pageAddr);
- final int indirectCnt = getIndirectCount(pageAddr);
-
- int curIndirectCnt = indirectCnt;
-
- assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items.
-
- // Remove the last item on the page.
- if (directCnt == 1) {
- assert (indirectCnt == 0 && itemId == 0) ||
- (indirectCnt == 1 && itemId == itemId(getItem(pageAddr, 1))) : itemId;
-
- setEmptyPage(pageAddr, pageSize);
- }
- else {
- // Get the entry size before the actual remove.
- int rmvEntrySize = getPageEntrySize(pageAddr, dataOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- int indirectId = 0;
-
- if (itemId >= directCnt) { // Need to remove indirect item.
- assert indirectCnt > 0;
-
- indirectId = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
-
- assert indirectId >= directCnt;
-
- itemId = directItemIndex(getItem(pageAddr, indirectId));
-
- assert itemId < directCnt;
- }
-
- boolean dropLast = true;
-
- if (itemId + 1 < directCnt) // It is not the last direct item.
- dropLast = moveLastItem(pageAddr, itemId, directCnt, indirectCnt);
-
- if (indirectId == 0) {// For the last direct item with no indirect item.
- if (dropLast)
- moveItems(pageAddr, directCnt, indirectCnt, -1, pageSize);
- else
- curIndirectCnt++;
- }
- else {
- if (dropLast)
- moveItems(pageAddr, directCnt, indirectId - directCnt, -1, pageSize);
-
- moveItems(pageAddr, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1, pageSize);
-
- if (dropLast)
- curIndirectCnt--;
- }
-
- setIndirectCount(pageAddr, curIndirectCnt);
- setDirectCount(pageAddr, directCnt - 1);
-
- assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
-
- // Increase free space.
- setRealFreeSpace(pageAddr,
- getRealFreeSpace(pageAddr) + rmvEntrySize + ITEM_SIZE * (directCnt - getDirectCount(pageAddr) + indirectCnt - getIndirectCount(pageAddr)),
- pageSize);
- }
-
- return nextLink;
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @param cnt Count.
- * @param step Step.
- * @param pageSize Page size.
- */
- private void moveItems(long pageAddr, int idx, int cnt, int step, int pageSize) {
- assert cnt >= 0: cnt;
-
- if (cnt != 0)
- moveBytes(pageAddr, itemOffset(idx), cnt * ITEM_SIZE, step * ITEM_SIZE, pageSize);
- }
-
- /**
- * @param newEntryFullSize New entry full size (with item, length and link).
- * @param firstEntryOff First entry data offset.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @return {@code true} If there is enough space for the entry.
- */
- private boolean isEnoughSpace(int newEntryFullSize, int firstEntryOff, int directCnt, int indirectCnt) {
- return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntryFullSize;
- }
-
- /**
- * Adds row to this data page and sets respective link to the given row object.
- *
- * @param pageAddr Page address.
- * @param row Cache data row.
- * @param rowSize Row size.
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void addRow(
- final long pageAddr,
- CacheDataRow row,
- final int rowSize,
- final int pageSize
- ) throws IgniteCheckedException {
- assert rowSize <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
-
- int fullEntrySize = getPageEntrySize(rowSize, SHOW_PAYLOAD_LEN | SHOW_ITEM);
-
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
-
- int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
- writeRowData(pageAddr, dataOff, rowSize, row, true);
-
- int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
- setLink(row, pageAddr, itemId);
- }
-
- /**
- * Adds row to this data page and sets respective link to the given row object.
- *
- * @param pageAddr Page address.
- * @param payload Payload.
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void addRow(
- long pageAddr,
- byte[] payload,
- int pageSize
- ) throws IgniteCheckedException {
- assert payload.length <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
-
- int fullEntrySize = getPageEntrySize(payload.length, SHOW_PAYLOAD_LEN | SHOW_ITEM);
-
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
-
- int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
- writeRowData(pageAddr, dataOff, payload);
-
- addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
- }
-
- /**
- * @param pageAddr Page address.
- * @param entryFullSize New entry full size (with item, length and link).
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param dataOff First entry offset.
- * @param pageSize Page size.
- * @return First entry offset after compaction.
- */
- private int compactIfNeed(
- final long pageAddr,
- final int entryFullSize,
- final int directCnt,
- final int indirectCnt,
- int dataOff,
- int pageSize
- ) {
- if (!isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt)) {
- dataOff = compactDataEntries(pageAddr, directCnt, pageSize);
-
- assert isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt);
- }
-
- return dataOff;
- }
-
- /**
- * Put item reference on entry.
- *
- * @param pageAddr Page address.
- * @param fullEntrySize Full entry size (with link, payload size and item).
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param dataOff Data offset.
- * @param pageSize Page size.
- * @return Item ID.
- */
- private int addItem(final long pageAddr,
- final int fullEntrySize,
- final int directCnt,
- final int indirectCnt,
- final int dataOff,
- final int pageSize)
- {
- setFirstEntryOffset(pageAddr, dataOff, pageSize);
-
- int itemId = insertItem(pageAddr, dataOff, directCnt, indirectCnt, pageSize);
-
- assert checkIndex(itemId): itemId;
- assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
-
- // Update free space. If number of indirect items changed, then we were able to reuse an item slot.
- setRealFreeSpace(pageAddr,
- getRealFreeSpace(pageAddr) - fullEntrySize + (getIndirectCount(pageAddr) != indirectCnt ? ITEM_SIZE : 0),
- pageSize);
-
- return itemId;
- }
-
- /**
- * @param pageAddr Page address.
- * @param fullEntrySize Full entry size.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param pageSize Page size.
- * @return Offset in the buffer where the entry must be written.
- */
- private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCnt, int indirectCnt, int pageSize) {
- int dataOff = getFirstEntryOffset(pageAddr);
-
- // Compact if we do not have enough space for entry.
- dataOff = compactIfNeed(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
- // We will write data right before the first entry.
- dataOff -= fullEntrySize - ITEM_SIZE;
-
- return dataOff;
- }
-
- /**
- * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
- *
- * @param pageMem Page memory.
- * @param pageAddr Page address.
- * @param row Cache data row.
- * @param written Number of bytes of row size that was already written.
- * @param rowSize Row size.
- * @param pageSize Page size.
- * @return Written payload size.
- * @throws IgniteCheckedException If failed.
- */
- public int addRowFragment(
- PageMemory pageMem,
- long pageAddr,
- CacheDataRow row,
- int written,
- int rowSize,
- int pageSize
- ) throws IgniteCheckedException {
- return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
- }
-
- /**
- * Adds this payload as a fragment to this data page.
- *
- * @param pageAddr Page address.
- * @param payload Payload bytes.
- * @param lastLink Link to the previous written fragment (link to the tail).
- * @param pageSize Page size.
- * @throws IgniteCheckedException If failed.
- */
- public void addRowFragment(
- long pageAddr,
- byte[] payload,
- long lastLink,
- int pageSize
- ) throws IgniteCheckedException {
- addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
- }
-
- /**
- * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
- *
- * @param pageMem Page memory.
- * @param pageAddr Page address.
- * @param written Number of bytes of row size that was already written.
- * @param rowSize Row size.
- * @param lastLink Link to the previous written fragment (link to the tail).
- * @param row Row.
- * @param payload Payload bytes.
- * @param pageSize Page size.
- * @return Written payload size.
- * @throws IgniteCheckedException If failed.
- */
- private int addRowFragment(
- PageMemory pageMem,
- long pageAddr,
- int written,
- int rowSize,
- long lastLink,
- CacheDataRow row,
- byte[] payload,
- int pageSize
- ) throws IgniteCheckedException {
- assert payload == null ^ row == null;
-
- int directCnt = getDirectCount(pageAddr);
- int indirectCnt = getIndirectCount(pageAddr);
-
- int payloadSize = payload != null ? payload.length :
- Math.min(rowSize - written, getFreeSpace(pageAddr));
-
- int fullEntrySize = getPageEntrySize(payloadSize, SHOW_PAYLOAD_LEN | SHOW_LINK | SHOW_ITEM);
- int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
-
- if (payload == null) {
- ByteBuffer buf = pageMem.pageBuffer(pageAddr);
-
- buf.position(dataOff);
-
- short p = (short)(payloadSize | FRAGMENTED_FLAG);
-
- buf.putShort(p);
- buf.putLong(lastLink);
-
- int rowOff = rowSize - written - payloadSize;
-
- writeFragmentData(row, buf, rowOff, payloadSize);
- }
- else {
- PageUtils.putShort(pageAddr, dataOff, (short)(payloadSize | FRAGMENTED_FLAG));
-
- PageUtils.putLong(pageAddr, dataOff + 2, lastLink);
-
- PageUtils.putBytes(pageAddr, dataOff + 10, payload);
- }
-
- int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
-
- if (row != null)
- setLink(row, pageAddr, itemId);
-
- return payloadSize;
- }
-
- /**
- * @param row Row to set link to.
- * @param pageAddr Page address.
- * @param itemId Item ID.
- */
- private void setLink(CacheDataRow row, long pageAddr, int itemId) {
- row.link(PageIdUtils.link(getPageId(pageAddr), itemId));
- }
-
- /**
- * Write row data fragment.
- *
- * @param row Row.
- * @param buf Byte buffer.
- * @param rowOff Offset in row data bytes.
- * @param payloadSize Data length that should be written in a fragment.
- * @throws IgniteCheckedException If failed.
- */
- private void writeFragmentData(
- final CacheDataRow row,
- final ByteBuffer buf,
- final int rowOff,
- final int payloadSize
- ) throws IgniteCheckedException {
- final int keySize = row.key().valueBytesLength(null);
- final int valSize = row.value().valueBytesLength(null);
-
- int written = writeFragment(row, buf, rowOff, payloadSize, EntryPart.CACHE_ID, keySize, valSize);
- written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.KEY, keySize, valSize);
- written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.EXPIRE_TIME, keySize, valSize);
- written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.VALUE, keySize, valSize);
- written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.VERSION, keySize, valSize);
-
- assert written == payloadSize;
- }
-
- /**
- * Try to write fragment data.
- *
- * @param rowOff Offset in row data bytes.
- * @param payloadSize Data length that should be written in this fragment.
- * @param type Type of the part of entry.
- * @return Actually written data.
- * @throws IgniteCheckedException If fail.
- */
- private int writeFragment(
- final CacheDataRow row,
- final ByteBuffer buf,
- final int rowOff,
- final int payloadSize,
- final EntryPart type,
- final int keySize,
- final int valSize
- ) throws IgniteCheckedException {
- if (payloadSize == 0)
- return 0;
-
- final int prevLen;
- final int curLen;
-
- int cacheIdSize = row.cacheId() == 0 ? 0 : 4;
-
- switch (type) {
- case CACHE_ID:
- prevLen = 0;
- curLen = cacheIdSize;
-
- break;
-
- case KEY:
- prevLen = cacheIdSize;
- curLen = cacheIdSize + keySize;
-
- break;
-
- case EXPIRE_TIME:
- prevLen = cacheIdSize + keySize;
- curLen = cacheIdSize + keySize + 8;
-
- break;
-
- case VALUE:
- prevLen = cacheIdSize + keySize + 8;
- curLen = cacheIdSize + keySize + valSize + 8;
-
- break;
-
- case VERSION:
- prevLen = cacheIdSize + keySize + valSize + 8;
- curLen = cacheIdSize + keySize + valSize + CacheVersionIO.size(row.version(), false) + 8;
-
- break;
-
- default:
- throw new IllegalArgumentException("Unknown entry part type: " + type);
- }
-
- if (curLen <= rowOff)
- return 0;
-
- final int len = Math.min(curLen - rowOff, payloadSize);
-
- if (type == EntryPart.EXPIRE_TIME)
- writeExpireTimeFragment(buf, row.expireTime(), rowOff, len, prevLen);
- else if (type == EntryPart.CACHE_ID)
- writeCacheIdFragment(buf, row.cacheId(), rowOff, len, prevLen);
- else if (type != EntryPart.VERSION) {
- // Write key or value.
- final CacheObject co = type == EntryPart.KEY ? row.key() : row.value();
-
- co.putValue(buf, rowOff - prevLen, len);
- }
- else
- writeVersionFragment(buf, row.version(), rowOff, len, prevLen);
-
- return len;
- }
-
- /**
- * @param buf Byte buffer.
- * @param ver Version.
- * @param rowOff Row offset.
- * @param len Length.
- * @param prevLen previous length.
- */
- private void writeVersionFragment(ByteBuffer buf, GridCacheVersion ver, int rowOff, int len, int prevLen) {
- int verSize = CacheVersionIO.size(ver, false);
-
- assert len <= verSize: len;
-
- if (verSize == len) { // Here we check for equality but not <= because version is the last.
- // Here we can write version directly.
- CacheVersionIO.write(buf, ver, false);
- }
- else {
- // We are in the middle of cache version.
- ByteBuffer verBuf = ByteBuffer.allocate(verSize);
-
- verBuf.order(buf.order());
-
- CacheVersionIO.write(verBuf, ver, false);
-
- buf.put(verBuf.array(), rowOff - prevLen, len);
- }
- }
-
- /**
- * @param buf Byte buffer.
- * @param expireTime Expire time.
- * @param rowOff Row offset.
- * @param len Length.
- * @param prevLen previous length.
- */
- private void writeExpireTimeFragment(ByteBuffer buf, long expireTime, int rowOff, int len, int prevLen) {
- int size = 8;
-
- if (size <= len)
- buf.putLong(expireTime);
- else {
- ByteBuffer timeBuf = ByteBuffer.allocate(size);
-
- timeBuf.order(buf.order());
-
- timeBuf.putLong(expireTime);
-
- buf.put(timeBuf.array(), rowOff - prevLen, len);
- }
- }
-
- /**
- * @param buf Buffer.
- * @param cacheId Cache ID.
- * @param rowOff Row offset.
- * @param len Length.
- * @param prevLen Prev length.
- */
- private void writeCacheIdFragment(ByteBuffer buf, int cacheId, int rowOff, int len, int prevLen) {
- if (cacheId == 0)
- return;
-
- int size = 4;
-
- if (size <= len)
- buf.putInt(cacheId);
- else {
- ByteBuffer cacheIdBuf = ByteBuffer.allocate(size);
-
- cacheIdBuf.order(buf.order());
-
- cacheIdBuf.putInt(cacheId);
-
- buf.put(cacheIdBuf.array(), rowOff - prevLen, len);
- }
- }
-
- /**
- *
- */
- private enum EntryPart {
- /** */
- KEY,
-
- /** */
- VALUE,
-
- /** */
- VERSION,
-
- /** */
- EXPIRE_TIME,
-
- /** */
- CACHE_ID
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param directCnt Direct items count.
- * @param indirectCnt Indirect items count.
- * @param pageSize Page size.
- * @return Item ID (insertion index).
- */
- private int insertItem(long pageAddr, int dataOff, int directCnt, int indirectCnt, int pageSize) {
- if (indirectCnt > 0) {
- // If the first indirect item is on correct place to become the last direct item, do the transition
- // and insert the new item into the free slot which was referenced by this first indirect item.
- short item = getItem(pageAddr, directCnt);
-
- if (itemId(item) == directCnt) {
- int directItemIdx = directItemIndex(item);
-
- setItem(pageAddr, directCnt, getItem(pageAddr, directItemIdx));
- setItem(pageAddr, directItemIdx, directItemFromOffset(dataOff));
-
- setDirectCount(pageAddr, directCnt + 1);
- setIndirectCount(pageAddr, indirectCnt - 1);
-
- return directItemIdx;
- }
- }
-
- // Move all the indirect items forward to make a free slot and insert new item at the end of direct items.
- moveItems(pageAddr, directCnt, indirectCnt, +1, pageSize);
-
- setItem(pageAddr, directCnt, directItemFromOffset(dataOff));
-
- setDirectCount(pageAddr, directCnt + 1);
- assert getDirectCount(pageAddr) == directCnt + 1;
-
- return directCnt; // Previous directCnt will be our itemId.
- }
-
- /**
- * @param pageAddr Page address.
- * @param directCnt Direct items count.
- * @param pageSize Page size.
- * @return New first entry offset.
- */
- private int compactDataEntries(long pageAddr, int directCnt, int pageSize) {
- assert checkCount(directCnt): directCnt;
-
- int[] offs = new int[directCnt];
-
- for (int i = 0; i < directCnt; i++) {
- int off = directItemToOffset(getItem(pageAddr, i));
-
- offs[i] = (off << 8) | i; // This way we'll be able to sort by offset using Arrays.sort(...).
- }
-
- Arrays.sort(offs);
-
- // Move right all of the entries if possible to make the page as compact as possible to its tail.
- int prevOff = pageSize;
-
- final int start = directCnt - 1;
- int curOff = offs[start] >>> 8;
- int curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- for (int i = start; i >= 0; i--) {
- assert curOff < prevOff : curOff;
-
- int delta = prevOff - (curOff + curEntrySize);
-
- int off = curOff;
- int entrySize = curEntrySize;
-
- if (delta != 0) { // Move right.
- assert delta > 0: delta;
-
- int itemId = offs[i] & 0xFF;
-
- setItem(pageAddr, itemId, directItemFromOffset(curOff + delta));
-
- for (int j = i - 1; j >= 0; j--) {
- int offNext = offs[j] >>> 8;
- int nextSize = getPageEntrySize(pageAddr, offNext, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- if (offNext + nextSize == off) {
- i--;
-
- off = offNext;
- entrySize += nextSize;
-
- itemId = offs[j] & 0xFF;
- setItem(pageAddr, itemId, directItemFromOffset(offNext + delta));
- }
- else {
- curOff = offNext;
- curEntrySize = nextSize;
-
- break;
- }
- }
-
- moveBytes(pageAddr, off, entrySize, delta, pageSize);
-
- off += delta;
- }
- else if (i > 0) {
- curOff = offs[i - 1] >>> 8;
- curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
- }
-
- prevOff = off;
- }
-
- return prevOff;
- }
-
- /**
- * Full-scan free space calculation procedure.
- *
- * @param pageAddr Page to scan.
- * @param pageSize Page size.
- * @return Actual free space in the buffer.
- */
- private int actualFreeSpace(long pageAddr, int pageSize) {
- int directCnt = getDirectCount(pageAddr);
-
- int entriesSize = 0;
-
- for (int i = 0; i < directCnt; i++) {
- int off = directItemToOffset(getItem(pageAddr, i));
-
- int entrySize = getPageEntrySize(pageAddr, off, SHOW_PAYLOAD_LEN | SHOW_LINK);
-
- entriesSize += entrySize;
- }
-
- return pageSize - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(pageAddr)) * ITEM_SIZE;
- }
-
- /**
- * @param addr Address.
- * @param off Offset.
- * @param cnt Count.
- * @param step Step.
- * @param pageSize Page size.
- */
- private void moveBytes(long addr, int off, int cnt, int step, int pageSize) {
- assert step != 0: step;
- assert off + step >= 0;
- assert off + step + cnt <= pageSize : "[off=" + off + ", step=" + step + ", cnt=" + cnt +
- ", cap=" + pageSize + ']';
-
- PageHandler.copyMemory(addr, addr, off, off + step, cnt);
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param payloadSize Payload size.
- * @param row Data row.
- * @param newRow {@code False} if existing cache entry is updated, in this case skip key data write.
- * @throws IgniteCheckedException If failed.
- */
- private void writeRowData(
- long pageAddr,
- int dataOff,
- int payloadSize,
- CacheDataRow row,
- boolean newRow
- ) throws IgniteCheckedException {
- long addr = pageAddr + dataOff;
-
- int cacheIdSize = row.cacheId() != 0 ? 4 : 0;
-
- if (newRow) {
- PageUtils.putShort(addr, 0, (short)payloadSize);
- addr += 2;
-
- if (cacheIdSize != 0) {
- PageUtils.putInt(addr, 0, row.cacheId());
-
- addr += cacheIdSize;
- }
-
- addr += row.key().putValue(addr);
- }
- else
- addr += (2 + cacheIdSize + row.key().valueBytesLength(null));
-
- addr += row.value().putValue(addr);
-
- CacheVersionIO.write(addr, row.version(), false);
- addr += CacheVersionIO.size(row.version(), false);
-
- PageUtils.putLong(addr, 0, row.expireTime());
- }
-
- /**
- * @param pageAddr Page address.
- * @param dataOff Data offset.
- * @param payload Payload
- */
- private void writeRowData(
- long pageAddr,
- int dataOff,
- byte[] payload
- ) {
- PageUtils.putShort(pageAddr, dataOff, (short)payload.length);
- dataOff += 2;
-
- PageUtils.putBytes(pageAddr, dataOff, payload);
- }
-
- /**
- * Defines closure interface for applying computations to data page items.
- *
- * @param <T> Closure return type.
- */
- public interface CC<T> {
- /**
- * Closure body.
- *
- * @param link Link to item.
- * @return Closure return value.
- * @throws IgniteCheckedException In case of error in closure body.
- */
- public T apply(long link) throws IgniteCheckedException;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPagePayload.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPagePayload.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPagePayload.java
deleted file mode 100644
index 277bdc7..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/tree/io/DataPagePayload.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.tree.io;
-
-/**
- *
- */
-public class DataPagePayload {
- /** */
- private final int off;
-
- /** */
- private final int payloadSize;
-
- /** */
- private final long nextLink;
-
- /**
- * @param off Offset.
- * @param payloadSize Payload size.
- * @param nextLink Next link.
- */
- DataPagePayload(int off, int payloadSize, long nextLink) {
- this.off = off;
- this.payloadSize = payloadSize;
- this.nextLink = nextLink;
- }
-
- /**
- * @return Offset.
- */
- public int offset() {
- return off;
- }
-
- /**
- * @return Payload size.
- */
- public int payloadSize() {
- return payloadSize;
- }
-
- /**
- * @return Link to the next fragment or {@code 0} if it is the last fragment or the data row is not fragmented.
- */
- public long nextLink() {
- return nextLink;
- }
-}
[09/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/serializer/RecordV1Serializer.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/serializer/RecordV1Serializer.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/serializer/RecordV1Serializer.java
deleted file mode 100644
index ce66b97..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/serializer/RecordV1Serializer.java
+++ /dev/null
@@ -1,1654 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal.serializer;
-
-import java.io.DataInput;
-import java.io.EOFException;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.CacheState;
-import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
-import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.LazyDataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
-import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord.RecordType;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageSetFreeListPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.FixLeftmostChildRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.FixRemoveId;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InnerReplaceRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MergeRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageAddRootRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastAllocatedIndex;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulFullSnapshotId;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateLastSuccessfulSnapshotId;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListInitNewPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListRemovePageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetNextRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.SplitForwardPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.TrackingPageDeltaRecord;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheOperation;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.CacheVersionIO;
-import org.apache.ignite.internal.processors.cache.database.wal.ByteBufferBackedDataInput;
-import org.apache.ignite.internal.processors.cache.database.wal.FileInput;
-import org.apache.ignite.internal.processors.cache.database.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.database.wal.RecordSerializer;
-import org.apache.ignite.internal.processors.cache.database.wal.SegmentEofException;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.PureJavaCrc32;
-import org.apache.ignite.internal.processors.cache.database.wal.record.HeaderRecord;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
-
-/**
- * Record V1 serializer.
- */
-public class RecordV1Serializer implements RecordSerializer {
- /** */
- public static final int HEADER_RECORD_SIZE = /*Type*/1 + /*Pointer */12 + /*Magic*/8 + /*Version*/4 + /*CRC*/4;
-
- /** */
- private GridCacheSharedContext cctx;
-
- /** */
- private int pageSize;
-
- /** */
- private IgniteCacheObjectProcessor co;
-
- /** */
- private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
-
- /**
- * @param cctx Cache shared context.
- */
- public RecordV1Serializer(GridCacheSharedContext cctx) {
- this.cctx = cctx;
-
- co = cctx.kernalContext().cacheObjects();
- pageSize = cctx.database().pageSize();
- }
-
- /** {@inheritDoc} */
- @Override public int version() {
- return 1;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("CastConflictsWithInstanceof")
- @Override public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException {
- assert record.size() > 0 && buf.remaining() >= record.size() : record.size();
-
- int startPos = buf.position();
-
- buf.put((byte)(record.type().ordinal() + 1));
-
- putPosition(buf, (FileWALPointer)record.position());
-
- switch (record.type()) {
- case PAGE_RECORD:
- PageSnapshot snap = (PageSnapshot)record;
-
- buf.putInt(snap.fullPageId().cacheId());
- buf.putLong(snap.fullPageId().pageId());
- buf.put(snap.pageData());
-
- break;
-
- case MEMORY_RECOVERY:
- MemoryRecoveryRecord memoryRecoveryRecord = (MemoryRecoveryRecord)record;
-
- buf.putLong(memoryRecoveryRecord.time());
-
- break;
-
- case PARTITION_DESTROY:
- PartitionDestroyRecord partDestroy = (PartitionDestroyRecord)record;
-
- buf.putInt(partDestroy.cacheId());
- buf.putInt(partDestroy.partitionId());
-
- break;
-
- case META_PAGE_INIT:
- MetaPageInitRecord updRootsRec = (MetaPageInitRecord)record;
-
- buf.putInt(updRootsRec.cacheId());
- buf.putLong(updRootsRec.pageId());
-
- buf.putShort((short)updRootsRec.ioType());
- buf.putShort((short)updRootsRec.ioVersion());
- buf.putLong(updRootsRec.treeRoot());
- buf.putLong(updRootsRec.reuseListRoot());
-
- break;
-
- case PARTITION_META_PAGE_UPDATE_COUNTERS:
- MetaPageUpdatePartitionDataRecord partDataRec = (MetaPageUpdatePartitionDataRecord)record;
-
- buf.putInt(partDataRec.cacheId());
- buf.putLong(partDataRec.pageId());
-
- buf.putLong(partDataRec.updateCounter());
- buf.putLong(partDataRec.globalRemoveId());
- buf.putInt(partDataRec.partitionSize());
- buf.putLong(partDataRec.countersPageId());
- buf.put(partDataRec.state());
- buf.putInt(partDataRec.allocatedIndexCandidate());
-
- break;
-
- case CHECKPOINT_RECORD:
- CheckpointRecord cpRec = (CheckpointRecord)record;
-
- assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
- "Invalid WAL record: " + cpRec;
-
- FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
- UUID cpId = cpRec.checkpointId();
-
- buf.putLong(cpId.getMostSignificantBits());
- buf.putLong(cpId.getLeastSignificantBits());
-
- buf.put(walPtr == null ? (byte)0 : 1);
-
- if (walPtr != null) {
- buf.putLong(walPtr.index());
- buf.putInt(walPtr.fileOffset());
- buf.putInt(walPtr.length());
- }
-
- putCacheStates(buf, cpRec.cacheGroupStates());
-
- buf.put(cpRec.end() ? (byte)1 : 0);
-
- break;
-
- case DATA_RECORD:
- DataRecord dataRec = (DataRecord)record;
-
- buf.putInt(dataRec.writeEntries().size());
-
- for (DataEntry dataEntry : dataRec.writeEntries())
- putDataEntry(buf, dataEntry);
-
- break;
-
- case HEADER_RECORD:
- buf.putLong(HeaderRecord.MAGIC);
-
- buf.putInt(((HeaderRecord)record).version());
-
- break;
-
- case DATA_PAGE_INSERT_RECORD:
- DataPageInsertRecord diRec = (DataPageInsertRecord)record;
-
- buf.putInt(diRec.cacheId());
- buf.putLong(diRec.pageId());
-
- buf.putShort((short)diRec.payload().length);
-
- buf.put(diRec.payload());
-
- break;
-
- case DATA_PAGE_UPDATE_RECORD:
- DataPageUpdateRecord uRec = (DataPageUpdateRecord)record;
-
- buf.putInt(uRec.cacheId());
- buf.putLong(uRec.pageId());
- buf.putInt(uRec.itemId());
-
- buf.putShort((short)uRec.payload().length);
-
- buf.put(uRec.payload());
-
- break;
-
- case DATA_PAGE_INSERT_FRAGMENT_RECORD:
- final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record;
-
- buf.putInt(difRec.cacheId());
- buf.putLong(difRec.pageId());
-
- buf.putLong(difRec.lastLink());
- buf.putInt(difRec.payloadSize());
- buf.put(difRec.payload());
-
- break;
-
- case DATA_PAGE_REMOVE_RECORD:
- DataPageRemoveRecord drRec = (DataPageRemoveRecord)record;
-
- buf.putInt(drRec.cacheId());
- buf.putLong(drRec.pageId());
-
- buf.put((byte)drRec.itemId());
-
- break;
-
- case DATA_PAGE_SET_FREE_LIST_PAGE:
- DataPageSetFreeListPageRecord freeListRec = (DataPageSetFreeListPageRecord)record;
-
- buf.putInt(freeListRec.cacheId());
- buf.putLong(freeListRec.pageId());
-
- buf.putLong(freeListRec.freeListPage());
-
- break;
-
- case INIT_NEW_PAGE_RECORD:
- InitNewPageRecord inpRec = (InitNewPageRecord)record;
-
- buf.putInt(inpRec.cacheId());
- buf.putLong(inpRec.pageId());
-
- buf.putShort((short)inpRec.ioType());
- buf.putShort((short)inpRec.ioVersion());
- buf.putLong(inpRec.newPageId());
-
- break;
-
- case BTREE_META_PAGE_INIT_ROOT:
- MetaPageInitRootRecord imRec = (MetaPageInitRootRecord)record;
-
- buf.putInt(imRec.cacheId());
- buf.putLong(imRec.pageId());
-
- buf.putLong(imRec.rootId());
-
- break;
-
- case BTREE_META_PAGE_INIT_ROOT2:
- MetaPageInitRootInlineRecord imRec2 = (MetaPageInitRootInlineRecord)record;
-
- buf.putInt(imRec2.cacheId());
- buf.putLong(imRec2.pageId());
-
- buf.putLong(imRec2.rootId());
-
- buf.putShort((short)imRec2.inlineSize());
- break;
-
- case BTREE_META_PAGE_ADD_ROOT:
- MetaPageAddRootRecord arRec = (MetaPageAddRootRecord)record;
-
- buf.putInt(arRec.cacheId());
- buf.putLong(arRec.pageId());
-
- buf.putLong(arRec.rootId());
-
- break;
-
- case BTREE_META_PAGE_CUT_ROOT:
- MetaPageCutRootRecord crRec = (MetaPageCutRootRecord)record;
-
- buf.putInt(crRec.cacheId());
- buf.putLong(crRec.pageId());
-
- break;
-
- case BTREE_INIT_NEW_ROOT:
- NewRootInitRecord<?> riRec = (NewRootInitRecord<?>)record;
-
- buf.putInt(riRec.cacheId());
- buf.putLong(riRec.pageId());
-
- buf.putLong(riRec.rootId());
- buf.putShort((short)riRec.io().getType());
- buf.putShort((short)riRec.io().getVersion());
- buf.putLong(riRec.leftId());
- buf.putLong(riRec.rightId());
-
- putRow(buf, riRec.rowBytes());
-
- break;
-
- case BTREE_PAGE_RECYCLE:
- RecycleRecord recRec = (RecycleRecord)record;
-
- buf.putInt(recRec.cacheId());
- buf.putLong(recRec.pageId());
-
- buf.putLong(recRec.newPageId());
-
- break;
-
- case BTREE_PAGE_INSERT:
- InsertRecord<?> inRec = (InsertRecord<?>)record;
-
- buf.putInt(inRec.cacheId());
- buf.putLong(inRec.pageId());
-
- buf.putShort((short)inRec.io().getType());
- buf.putShort((short)inRec.io().getVersion());
- buf.putShort((short)inRec.index());
- buf.putLong(inRec.rightId());
-
- putRow(buf, inRec.rowBytes());
-
- break;
-
- case BTREE_FIX_LEFTMOST_CHILD:
- FixLeftmostChildRecord flRec = (FixLeftmostChildRecord)record;
-
- buf.putInt(flRec.cacheId());
- buf.putLong(flRec.pageId());
-
- buf.putLong(flRec.rightId());
-
- break;
-
- case BTREE_FIX_COUNT:
- FixCountRecord fcRec = (FixCountRecord)record;
-
- buf.putInt(fcRec.cacheId());
- buf.putLong(fcRec.pageId());
-
- buf.putShort((short)fcRec.count());
-
- break;
-
- case BTREE_PAGE_REPLACE:
- ReplaceRecord<?> rRec = (ReplaceRecord<?>)record;
-
- buf.putInt(rRec.cacheId());
- buf.putLong(rRec.pageId());
-
- buf.putShort((short)rRec.io().getType());
- buf.putShort((short)rRec.io().getVersion());
- buf.putShort((short)rRec.index());
-
- putRow(buf, rRec.rowBytes());
-
- break;
-
- case BTREE_PAGE_REMOVE:
- RemoveRecord rmRec = (RemoveRecord)record;
-
- buf.putInt(rmRec.cacheId());
- buf.putLong(rmRec.pageId());
-
- buf.putShort((short)rmRec.index());
- buf.putShort((short)rmRec.count());
-
- break;
-
- case BTREE_PAGE_INNER_REPLACE:
- InnerReplaceRecord<?> irRec = (InnerReplaceRecord<?>)record;
-
- buf.putInt(irRec.cacheId());
- buf.putLong(irRec.pageId());
-
- buf.putShort((short)irRec.destinationIndex());
- buf.putLong(irRec.sourcePageId());
- buf.putShort((short)irRec.sourceIndex());
- buf.putLong(irRec.removeId());
-
- break;
-
- case BTREE_FORWARD_PAGE_SPLIT:
- SplitForwardPageRecord sfRec = (SplitForwardPageRecord)record;
-
- buf.putInt(sfRec.cacheId());
- buf.putLong(sfRec.pageId());
-
- buf.putLong(sfRec.forwardId());
- buf.putShort((short)sfRec.ioType());
- buf.putShort((short)sfRec.ioVersion());
- buf.putLong(sfRec.sourcePageId());
- buf.putShort((short)sfRec.middleIndex());
- buf.putShort((short)sfRec.count());
-
- break;
-
- case BTREE_EXISTING_PAGE_SPLIT:
- SplitExistingPageRecord seRec = (SplitExistingPageRecord)record;
-
- buf.putInt(seRec.cacheId());
- buf.putLong(seRec.pageId());
-
- buf.putShort((short)seRec.middleIndex());
- buf.putLong(seRec.forwardId());
-
- break;
-
- case BTREE_PAGE_MERGE:
- MergeRecord<?> mRec = (MergeRecord<?>)record;
-
- buf.putInt(mRec.cacheId());
- buf.putLong(mRec.pageId());
-
- buf.putLong(mRec.parentId());
- buf.putShort((short)mRec.parentIndex());
- buf.putLong(mRec.rightId());
- buf.put((byte)(mRec.isEmptyBranch() ? 1 : 0));
-
- break;
-
- case PAGES_LIST_SET_NEXT:
- PagesListSetNextRecord plNextRec = (PagesListSetNextRecord)record;
-
- buf.putInt(plNextRec.cacheId());
- buf.putLong(plNextRec.pageId());
-
- buf.putLong(plNextRec.nextPageId());
-
- break;
-
- case PAGES_LIST_SET_PREVIOUS:
- PagesListSetPreviousRecord plPrevRec = (PagesListSetPreviousRecord)record;
-
- buf.putInt(plPrevRec.cacheId());
- buf.putLong(plPrevRec.pageId());
-
- buf.putLong(plPrevRec.previousPageId());
-
- break;
-
- case PAGES_LIST_INIT_NEW_PAGE:
- PagesListInitNewPageRecord plNewRec = (PagesListInitNewPageRecord)record;
-
- buf.putInt(plNewRec.cacheId());
- buf.putLong(plNewRec.pageId());
- buf.putInt(plNewRec.ioType());
- buf.putInt(plNewRec.ioVersion());
- buf.putLong(plNewRec.newPageId());
-
- buf.putLong(plNewRec.previousPageId());
- buf.putLong(plNewRec.dataPageId());
-
- break;
-
- case PAGES_LIST_ADD_PAGE:
- PagesListAddPageRecord plAddRec = (PagesListAddPageRecord)record;
-
- buf.putInt(plAddRec.cacheId());
- buf.putLong(plAddRec.pageId());
-
- buf.putLong(plAddRec.dataPageId());
-
- break;
-
- case PAGES_LIST_REMOVE_PAGE:
- PagesListRemovePageRecord plRmvRec = (PagesListRemovePageRecord)record;
-
- buf.putInt(plRmvRec.cacheId());
- buf.putLong(plRmvRec.pageId());
-
- buf.putLong(plRmvRec.removedPageId());
-
- break;
-
- case BTREE_FIX_REMOVE_ID:
- FixRemoveId frRec = (FixRemoveId)record;
-
- buf.putInt(frRec.cacheId());
- buf.putLong(frRec.pageId());
-
- buf.putLong(frRec.removeId());
-
- break;
-
- case TRACKING_PAGE_DELTA:
- TrackingPageDeltaRecord tpDelta = (TrackingPageDeltaRecord)record;
-
- buf.putInt(tpDelta.cacheId());
- buf.putLong(tpDelta.pageId());
-
- buf.putLong(tpDelta.pageIdToMark());
- buf.putLong(tpDelta.nextSnapshotId());
- buf.putLong(tpDelta.lastSuccessfulSnapshotId());
-
- break;
-
- case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
- MetaPageUpdateNextSnapshotId mpUpdateNextSnapshotId = (MetaPageUpdateNextSnapshotId)record;
-
- buf.putInt(mpUpdateNextSnapshotId.cacheId());
- buf.putLong(mpUpdateNextSnapshotId.pageId());
-
- buf.putLong(mpUpdateNextSnapshotId.nextSnapshotId());
-
- break;
-
- case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
- MetaPageUpdateLastSuccessfulFullSnapshotId mpUpdateLastSuccFullSnapshotId =
- (MetaPageUpdateLastSuccessfulFullSnapshotId)record;
-
- buf.putInt(mpUpdateLastSuccFullSnapshotId.cacheId());
- buf.putLong(mpUpdateLastSuccFullSnapshotId.pageId());
-
- buf.putLong(mpUpdateLastSuccFullSnapshotId.lastSuccessfulFullSnapshotId());
-
- break;
-
- case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
- MetaPageUpdateLastSuccessfulSnapshotId mpUpdateLastSuccSnapshotId =
- (MetaPageUpdateLastSuccessfulSnapshotId)record;
-
- buf.putInt(mpUpdateLastSuccSnapshotId.cacheId());
- buf.putLong(mpUpdateLastSuccSnapshotId.pageId());
-
- buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotId());
- buf.putLong(mpUpdateLastSuccSnapshotId.lastSuccessfulSnapshotTag());
-
- break;
-
- case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
- MetaPageUpdateLastAllocatedIndex mpUpdateLastAllocatedIdx =
- (MetaPageUpdateLastAllocatedIndex) record;
-
- buf.putInt(mpUpdateLastAllocatedIdx.cacheId());
- buf.putLong(mpUpdateLastAllocatedIdx.pageId());
-
- buf.putInt(mpUpdateLastAllocatedIdx.lastAllocatedIndex());
-
- break;
-
- case PART_META_UPDATE_STATE:
- PartitionMetaStateRecord partMetaStateRecord = (PartitionMetaStateRecord) record;
-
- buf.putInt(partMetaStateRecord.cacheId());
-
- buf.putInt(partMetaStateRecord.partitionId());
-
- buf.put(partMetaStateRecord.state());
-
- buf.putLong(partMetaStateRecord.updateCounter());
-
- break;
-
- case PAGE_LIST_META_RESET_COUNT_RECORD:
- PageListMetaResetCountRecord pageListMetaResetCntRecord = (PageListMetaResetCountRecord) record;
-
- buf.putInt(pageListMetaResetCntRecord.cacheId());
- buf.putLong(pageListMetaResetCntRecord.pageId());
-
- break;
-
- default:
- throw new UnsupportedOperationException("Type: " + record.type());
- }
-
- if (!skipCrc) {
- int curPos = buf.position();
-
- buf.position(startPos);
-
- // This call will move buffer position to the end of the record again.
- int crcVal = PureJavaCrc32.calcCrc32(buf, curPos - startPos);
-
- buf.putInt(crcVal);
- }
- else
- buf.putInt(0);
- }
-
- /** {@inheritDoc} */
- @Override public WALRecord readRecord(FileInput in0, WALPointer expPtr) throws IOException, IgniteCheckedException {
- long startPos = -1;
-
- try (FileInput.Crc32CheckingFileInput in = in0.startRead(skipCrc)) {
- startPos = in0.position();
-
- WALRecord res = readRecord(in, expPtr);
-
- assert res != null;
-
- res.size((int)(in0.position() - startPos + 4)); // Account for CRC which will be read afterwards.
-
- return res;
- }
- catch (EOFException | SegmentEofException e) {
- throw e;
- }
- catch (Exception e) {
- throw new IgniteCheckedException("Failed to read WAL record at position: " + startPos, e);
- }
- }
-
- /**
- * @param in In.
- */
- private WALRecord readRecord(ByteBufferBackedDataInput in, WALPointer expPtr) throws IOException, IgniteCheckedException {
- int type = in.readUnsignedByte();
-
- if (type == 0)
- throw new SegmentEofException("Reached logical end of the segment", null);
-
- FileWALPointer ptr = readPosition(in);
-
- if (!F.eq(ptr, expPtr))
- throw new SegmentEofException("WAL segment rollover detected (will end iteration) [expPtr=" + expPtr +
- ", readPtr=" + ptr + ']', null);
-
- RecordType recType = RecordType.fromOrdinal(type - 1);
-
- if (recType == null)
- throw new IOException("Unknown record type: " + type);
-
- WALRecord res;
-
- switch (recType) {
- case PAGE_RECORD:
- byte[] arr = new byte[pageSize];
-
- int cacheId = in.readInt();
- long pageId = in.readLong();
-
- in.readFully(arr);
-
- res = new PageSnapshot(new FullPageId(pageId, cacheId), arr);
-
- break;
-
- case CHECKPOINT_RECORD:
- long msb = in.readLong();
- long lsb = in.readLong();
- boolean hasPtr = in.readByte() != 0;
- int idx = hasPtr ? in.readInt() : 0;
- int offset = hasPtr ? in.readInt() : 0;
- int len = hasPtr ? in.readInt() : 0;
-
- Map<Integer, CacheState> states = readPartitionStates(in);
-
- boolean end = in.readByte() != 0;
-
- FileWALPointer walPtr = hasPtr ? new FileWALPointer(idx, offset, len) : null;
-
- CheckpointRecord cpRec = new CheckpointRecord(new UUID(msb, lsb), walPtr, end);
-
- cpRec.cacheGroupStates(states);
-
- res = cpRec;
-
- break;
-
- case META_PAGE_INIT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int ioType = in.readUnsignedShort();
- int ioVer = in.readUnsignedShort();
- long treeRoot = in.readLong();
- long reuseListRoot = in.readLong();
-
- res = new MetaPageInitRecord(cacheId, pageId, ioType, ioVer, treeRoot, reuseListRoot);
-
- break;
-
- case PARTITION_META_PAGE_UPDATE_COUNTERS:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long updCntr = in.readLong();
- long rmvId = in.readLong();
- int partSize = in.readInt();
- long countersPageId = in.readLong();
- byte state = in.readByte();
- int allocatedIdxCandidate = in.readInt();
-
- res = new MetaPageUpdatePartitionDataRecord(cacheId, pageId, updCntr, rmvId, partSize, countersPageId, state, allocatedIdxCandidate);
-
- break;
-
- case MEMORY_RECOVERY:
- long ts = in.readLong();
-
- res = new MemoryRecoveryRecord(ts);
-
- break;
-
- case PARTITION_DESTROY:
- cacheId = in.readInt();
- int partId = in.readInt();
-
- res = new PartitionDestroyRecord(cacheId, partId);
-
- break;
-
- case DATA_RECORD:
- int entryCnt = in.readInt();
-
- List<DataEntry> entries = new ArrayList<>(entryCnt);
-
- for (int i = 0; i < entryCnt; i++)
- entries.add(readDataEntry(in));
-
- res = new DataRecord(entries);
-
- break;
-
- case HEADER_RECORD:
- long magic = in.readLong();
-
- if (magic != HeaderRecord.MAGIC)
- throw new EOFException("Magic is corrupted [exp=" + U.hexLong(HeaderRecord.MAGIC) +
- ", actual=" + U.hexLong(magic) + ']');
-
- int ver = in.readInt();
-
- res = new HeaderRecord(ver);
-
- break;
-
- case DATA_PAGE_INSERT_RECORD: {
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int size = in.readUnsignedShort();
-
- in.ensure(size);
-
- byte[] payload = new byte[size];
-
- in.readFully(payload);
-
- res = new DataPageInsertRecord(cacheId, pageId, payload);
-
- break;
- }
-
- case DATA_PAGE_UPDATE_RECORD: {
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int itemId = in.readInt();
-
- int size = in.readUnsignedShort();
-
- in.ensure(size);
-
- byte[] payload = new byte[size];
-
- in.readFully(payload);
-
- res = new DataPageUpdateRecord(cacheId, pageId, itemId, payload);
-
- break;
- }
-
- case DATA_PAGE_INSERT_FRAGMENT_RECORD: {
- cacheId = in.readInt();
- pageId = in.readLong();
-
- final long lastLink = in.readLong();
- final int payloadSize = in.readInt();
-
- final byte[] payload = new byte[payloadSize];
-
- in.readFully(payload);
-
- res = new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink);
-
- break;
- }
-
- case DATA_PAGE_REMOVE_RECORD:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int itemId = in.readUnsignedByte();
-
- res = new DataPageRemoveRecord(cacheId, pageId, itemId);
-
- break;
-
- case DATA_PAGE_SET_FREE_LIST_PAGE:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long freeListPage = in.readLong();
-
- res = new DataPageSetFreeListPageRecord(cacheId, pageId, freeListPage);
-
- break;
-
- case INIT_NEW_PAGE_RECORD:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- ioType = in.readUnsignedShort();
- ioVer = in.readUnsignedShort();
- long virtualPageId = in.readLong();
-
- res = new InitNewPageRecord(cacheId, pageId, ioType, ioVer, virtualPageId);
-
- break;
-
- case BTREE_META_PAGE_INIT_ROOT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long rootId = in.readLong();
-
- res = new MetaPageInitRootRecord(cacheId, pageId, rootId);
-
- break;
-
- case BTREE_META_PAGE_INIT_ROOT2:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long rootId2 = in.readLong();
- int inlineSize = in.readShort();
-
- res = new MetaPageInitRootInlineRecord(cacheId, pageId, rootId2, inlineSize);
-
- break;
-
- case BTREE_META_PAGE_ADD_ROOT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- rootId = in.readLong();
-
- res = new MetaPageAddRootRecord(cacheId, pageId, rootId);
-
- break;
-
- case BTREE_META_PAGE_CUT_ROOT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- res = new MetaPageCutRootRecord(cacheId, pageId);
-
- break;
-
- case BTREE_INIT_NEW_ROOT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- rootId = in.readLong();
- ioType = in.readUnsignedShort();
- ioVer = in.readUnsignedShort();
- long leftId = in.readLong();
- long rightId = in.readLong();
-
- BPlusIO<?> io = BPlusIO.getBPlusIO(ioType, ioVer);
-
- byte[] rowBytes = new byte[io.getItemSize()];
-
- in.readFully(rowBytes);
-
- res = new NewRootInitRecord<>(cacheId, pageId, rootId, (BPlusInnerIO<?>)io, leftId, rowBytes, rightId);
-
- break;
-
- case BTREE_PAGE_RECYCLE:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long newPageId = in.readLong();
-
- res = new RecycleRecord(cacheId, pageId, newPageId);
-
- break;
-
- case BTREE_PAGE_INSERT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- ioType = in.readUnsignedShort();
- ioVer = in.readUnsignedShort();
- int itemIdx = in.readUnsignedShort();
- rightId = in.readLong();
-
- io = BPlusIO.getBPlusIO(ioType, ioVer);
-
- rowBytes = new byte[io.getItemSize()];
-
- in.readFully(rowBytes);
-
- res = new InsertRecord<>(cacheId, pageId, io, itemIdx, rowBytes, rightId);
-
- break;
-
- case BTREE_FIX_LEFTMOST_CHILD:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- rightId = in.readLong();
-
- res = new FixLeftmostChildRecord(cacheId, pageId, rightId);
-
- break;
-
- case BTREE_FIX_COUNT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int cnt = in.readUnsignedShort();
-
- res = new FixCountRecord(cacheId, pageId, cnt);
-
- break;
-
- case BTREE_PAGE_REPLACE:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- ioType = in.readUnsignedShort();
- ioVer = in.readUnsignedShort();
- itemIdx = in.readUnsignedShort();
-
- io = BPlusIO.getBPlusIO(ioType, ioVer);
-
- rowBytes = new byte[io.getItemSize()];
-
- in.readFully(rowBytes);
-
- res = new ReplaceRecord<>(cacheId, pageId, io, rowBytes, itemIdx);
-
- break;
-
- case BTREE_PAGE_REMOVE:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- itemIdx = in.readUnsignedShort();
- cnt = in.readUnsignedShort();
-
- res = new RemoveRecord(cacheId, pageId, itemIdx, cnt);
-
- break;
-
- case BTREE_PAGE_INNER_REPLACE:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int dstIdx = in.readUnsignedShort();
- long srcPageId = in.readLong();
- int srcIdx = in.readUnsignedShort();
- rmvId = in.readLong();
-
- res = new InnerReplaceRecord<>(cacheId, pageId, dstIdx, srcPageId, srcIdx, rmvId);
-
- break;
-
- case BTREE_FORWARD_PAGE_SPLIT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long fwdId = in.readLong();
- ioType = in.readUnsignedShort();
- ioVer = in.readUnsignedShort();
- srcPageId = in.readLong();
- int mid = in.readUnsignedShort();
- cnt = in.readUnsignedShort();
-
- res = new SplitForwardPageRecord(cacheId, pageId, fwdId, ioType, ioVer, srcPageId, mid, cnt);
-
- break;
-
- case BTREE_EXISTING_PAGE_SPLIT:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- mid = in.readUnsignedShort();
- fwdId = in.readLong();
-
- res = new SplitExistingPageRecord(cacheId, pageId, mid, fwdId);
-
- break;
-
- case BTREE_PAGE_MERGE:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long prntId = in.readLong();
- int prntIdx = in.readUnsignedShort();
- rightId = in.readLong();
- boolean emptyBranch = in.readBoolean();
-
- res = new MergeRecord<>(cacheId, pageId, prntId, prntIdx, rightId, emptyBranch);
-
- break;
-
- case BTREE_FIX_REMOVE_ID:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- rmvId = in.readLong();
-
- res = new FixRemoveId(cacheId, pageId, rmvId);
-
- break;
-
- case PAGES_LIST_SET_NEXT:
- cacheId = in.readInt();
- pageId = in.readLong();
- long nextPageId = in.readLong();
-
- res = new PagesListSetNextRecord(cacheId, pageId, nextPageId);
-
- break;
-
- case PAGES_LIST_SET_PREVIOUS:
- cacheId = in.readInt();
- pageId = in.readLong();
- long prevPageId = in.readLong();
-
- res = new PagesListSetPreviousRecord(cacheId, pageId, prevPageId);
-
- break;
-
- case PAGES_LIST_INIT_NEW_PAGE:
- cacheId = in.readInt();
- pageId = in.readLong();
- ioType = in.readInt();
- ioVer = in.readInt();
- newPageId = in.readLong();
- prevPageId = in.readLong();
- long addDataPageId = in.readLong();
-
- res = new PagesListInitNewPageRecord(cacheId, pageId, ioType, ioVer, newPageId, prevPageId, addDataPageId);
-
- break;
-
- case PAGES_LIST_ADD_PAGE:
- cacheId = in.readInt();
- pageId = in.readLong();
- long dataPageId = in.readLong();
-
- res = new PagesListAddPageRecord(cacheId, pageId, dataPageId);
-
- break;
-
- case PAGES_LIST_REMOVE_PAGE:
- cacheId = in.readInt();
- pageId = in.readLong();
- long rmvdPageId = in.readLong();
-
- res = new PagesListRemovePageRecord(cacheId, pageId, rmvdPageId);
-
- break;
-
- case TRACKING_PAGE_DELTA:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long pageIdToMark = in.readLong();
- long nextSnapshotId0 = in.readLong();
- long lastSuccessfulSnapshotId0 = in.readLong();
-
- res = new TrackingPageDeltaRecord(cacheId, pageId, pageIdToMark, nextSnapshotId0, lastSuccessfulSnapshotId0);
-
- break;
-
- case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long nextSnapshotId = in.readLong();
-
- res = new MetaPageUpdateNextSnapshotId(cacheId, pageId, nextSnapshotId);
-
- break;
-
- case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long lastSuccessfulFullSnapshotId = in.readLong();
-
- res = new MetaPageUpdateLastSuccessfulFullSnapshotId(cacheId, pageId, lastSuccessfulFullSnapshotId);
-
- break;
-
- case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- long lastSuccessfulSnapshotId = in.readLong();
- long lastSuccessfulSnapshotTag = in.readLong();
-
- res = new MetaPageUpdateLastSuccessfulSnapshotId(cacheId, pageId, lastSuccessfulSnapshotId, lastSuccessfulSnapshotTag);
-
- break;
-
- case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- int lastAllocatedIdx = in.readInt();
-
- res = new MetaPageUpdateLastAllocatedIndex(cacheId, pageId, lastAllocatedIdx);
-
- break;
-
- case PART_META_UPDATE_STATE:
- cacheId = in.readInt();
- partId = in.readInt();
-
- state = in.readByte();
-
- long updateCounter = in.readLong();
-
- res = new PartitionMetaStateRecord(cacheId, partId, GridDhtPartitionState.fromOrdinal(state), updateCounter);
-
- break;
-
- case PAGE_LIST_META_RESET_COUNT_RECORD:
- cacheId = in.readInt();
- pageId = in.readLong();
-
- res = new PageListMetaResetCountRecord(cacheId, pageId);
- break;
-
- case SWITCH_SEGMENT_RECORD:
- throw new EOFException("END OF SEGMENT");
-
- default:
- throw new UnsupportedOperationException("Type: " + recType);
- }
-
- return res;
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("CastConflictsWithInstanceof")
- @Override public int size(WALRecord record) throws IgniteCheckedException {
- int commonFields = /* Type */1 + /* Pointer */12 + /*CRC*/4;
-
- switch (record.type()) {
- case PAGE_RECORD:
- assert record instanceof PageSnapshot;
-
- PageSnapshot pageRec = (PageSnapshot)record;
-
- return commonFields + pageRec.pageData().length + 12;
-
- case CHECKPOINT_RECORD:
- CheckpointRecord cpRec = (CheckpointRecord)record;
-
- assert cpRec.checkpointMark() == null || cpRec.checkpointMark() instanceof FileWALPointer :
- "Invalid WAL record: " + cpRec;
-
- int cacheStatesSize = cacheStatesSize(cpRec.cacheGroupStates());
-
- FileWALPointer walPtr = (FileWALPointer)cpRec.checkpointMark();
-
- return commonFields + 18 + cacheStatesSize + (walPtr == null ? 0 : 16);
-
- case META_PAGE_INIT:
- return commonFields + /*cache ID*/4 + /*page ID*/8 + /*ioType*/2 + /*ioVer*/2 + /*tree root*/8 + /*reuse root*/8;
-
- case PARTITION_META_PAGE_UPDATE_COUNTERS:
- return commonFields + /*cache ID*/4 + /*page ID*/8 + /*upd cntr*/8 + /*rmv id*/8 + /*part size*/4 + /*counters page id*/8 + /*state*/ 1
- + /*allocatedIdxCandidate*/ 4;
-
- case MEMORY_RECOVERY:
- return commonFields + 8;
-
- case PARTITION_DESTROY:
- return commonFields + /*cacheId*/4 + /*partId*/4;
-
- case DATA_RECORD:
- DataRecord dataRec = (DataRecord)record;
-
- return commonFields + 4 + dataSize(dataRec);
-
- case HEADER_RECORD:
- return HEADER_RECORD_SIZE;
-
- case DATA_PAGE_INSERT_RECORD:
- DataPageInsertRecord diRec = (DataPageInsertRecord)record;
-
- return commonFields + 4 + 8 + 2 + diRec.payload().length;
-
- case DATA_PAGE_UPDATE_RECORD:
- DataPageUpdateRecord uRec = (DataPageUpdateRecord)record;
-
- return commonFields + 4 + 8 + 2 + 4 +
- uRec.payload().length;
-
- case DATA_PAGE_INSERT_FRAGMENT_RECORD:
- final DataPageInsertFragmentRecord difRec = (DataPageInsertFragmentRecord)record;
-
- return commonFields + 4 + 8 + 8 + 4 + difRec.payloadSize();
-
- case DATA_PAGE_REMOVE_RECORD:
- return commonFields + 4 + 8 + 1;
-
- case DATA_PAGE_SET_FREE_LIST_PAGE:
- return commonFields + 4 + 8 + 8;
-
- case INIT_NEW_PAGE_RECORD:
- return commonFields + 4 + 8 + 2 + 2 + 8;
-
- case BTREE_META_PAGE_INIT_ROOT:
- return commonFields + 4 + 8 + 8;
-
- case BTREE_META_PAGE_INIT_ROOT2:
- return commonFields + 4 + 8 + 8 + 2;
-
- case BTREE_META_PAGE_ADD_ROOT:
- return commonFields + 4 + 8 + 8;
-
- case BTREE_META_PAGE_CUT_ROOT:
- return commonFields + 4 + 8;
-
- case BTREE_INIT_NEW_ROOT:
- NewRootInitRecord<?> riRec = (NewRootInitRecord<?>)record;
-
- return commonFields + 4 + 8 + 8 + 2 + 2 + 8 + 8 + riRec.io().getItemSize();
-
- case BTREE_PAGE_RECYCLE:
- return commonFields + 4 + 8 + 8;
-
- case BTREE_PAGE_INSERT:
- InsertRecord<?> inRec = (InsertRecord<?>)record;
-
- return commonFields + 4 + 8 + 2 + 2 + 2 + 8 + inRec.io().getItemSize();
-
- case BTREE_FIX_LEFTMOST_CHILD:
- return commonFields + 4 + 8 + 8;
-
- case BTREE_FIX_COUNT:
- return commonFields + 4 + 8 + 2;
-
- case BTREE_PAGE_REPLACE:
- ReplaceRecord<?> rRec = (ReplaceRecord<?>)record;
-
- return commonFields + 4 + 8 + 2 + 2 + 2 + rRec.io().getItemSize();
-
- case BTREE_PAGE_REMOVE:
- return commonFields + 4 + 8 + 2 + 2;
-
- case BTREE_PAGE_INNER_REPLACE:
- return commonFields + 4 + 8 + 2 + 8 + 2 + 8;
-
- case BTREE_FORWARD_PAGE_SPLIT:
- return commonFields + 4 + 8 + 8 + 2 + 2 + 8 + 2 + 2;
-
- case BTREE_EXISTING_PAGE_SPLIT:
- return commonFields + 4 + 8 + 2 + 8;
-
- case BTREE_PAGE_MERGE:
- return commonFields + 4 + 8 + 8 + 2 + 8 + 1;
-
- case BTREE_FIX_REMOVE_ID:
- return commonFields + 4 + 8 + 8;
-
- case PAGES_LIST_SET_NEXT:
- return commonFields + 4 + 8 + 8;
-
- case PAGES_LIST_SET_PREVIOUS:
- return commonFields + 4 + 8 + 8;
-
- case PAGES_LIST_INIT_NEW_PAGE:
- return commonFields + 4 + 8 + 4 + 4 + 8 + 8 + 8;
-
- case PAGES_LIST_ADD_PAGE:
- return commonFields + 4 + 8 + 8;
-
- case PAGES_LIST_REMOVE_PAGE:
- return commonFields + 4 + 8 + 8;
-
- case TRACKING_PAGE_DELTA:
- return commonFields + 4 + 8 + 8 + 8 + 8;
-
- case META_PAGE_UPDATE_LAST_SUCCESSFUL_SNAPSHOT_ID:
- return commonFields + 4 + 8 + 8 + 8;
-
- case META_PAGE_UPDATE_LAST_SUCCESSFUL_FULL_SNAPSHOT_ID:
- return commonFields + 4 + 8 + 8;
-
- case META_PAGE_UPDATE_NEXT_SNAPSHOT_ID:
- return commonFields + 4 + 8 + 8;
-
- case META_PAGE_UPDATE_LAST_ALLOCATED_INDEX:
- return commonFields + 4 + 8 + 4;
-
- case PART_META_UPDATE_STATE:
- return commonFields + /*cacheId*/ 4 + /*partId*/ 4 + /*State*/1 + /*Update Counter*/ 8;
-
- case PAGE_LIST_META_RESET_COUNT_RECORD:
- return commonFields + /*cacheId*/ 4 + /*pageId*/ 8;
-
- case SWITCH_SEGMENT_RECORD:
- return commonFields;
-
- default:
- throw new UnsupportedOperationException("Type: " + record.type());
- }
- }
-
- /**
- * @param buf Byte buffer to serialize version to.
- * @param ptr File WAL pointer to write.
- */
- private void putPosition(ByteBuffer buf, FileWALPointer ptr) {
- buf.putLong(ptr.index());
- buf.putInt(ptr.fileOffset());
- }
-
- /**
- * @param in Data input to read pointer from.
- * @return Read file WAL pointer.
- * @throws IOException If failed to write.
- */
- private FileWALPointer readPosition(DataInput in) throws IOException {
- long idx = in.readLong();
- int fileOffset = in.readInt();
-
- return new FileWALPointer(idx, fileOffset, 0);
- }
-
- /**
- * @param dataRec Data record to serialize.
- * @return Full data record size.
- * @throws IgniteCheckedException If failed to obtain the length of one of the entries.
- */
- private int dataSize(DataRecord dataRec) throws IgniteCheckedException {
- int sz = 0;
-
- for (DataEntry entry : dataRec.writeEntries())
- sz += entrySize(entry);
-
- return sz;
- }
-
- /**
- * @param entry Entry to get size for.
- * @return Entry size.
- * @throws IgniteCheckedException If failed to get key or value bytes length.
- */
- private int entrySize(DataEntry entry) throws IgniteCheckedException {
- GridCacheContext cctx = this.cctx.cacheContext(entry.cacheId());
- CacheObjectContext coCtx = cctx.cacheObjectContext();
-
- return
- /*cache ID*/4 +
- /*key*/entry.key().valueBytesLength(coCtx) +
- /*value*/(entry.value() == null ? 4 : entry.value().valueBytesLength(coCtx)) +
- /*op*/1 +
- /*near xid ver*/CacheVersionIO.size(entry.nearXidVersion(), true) +
- /*write ver*/CacheVersionIO.size(entry.writeVersion(), false) +
- /*part ID*/4 +
- /*expire Time*/8 +
- /*part cnt*/8;
- }
-
- /**
- * @param states Partition states.
- * @return Size required to write partition states.
- */
- private int cacheStatesSize(Map<Integer, CacheState> states) {
- // Need 4 bytes for the number of caches.
- int size = 2;
-
- for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
- // Cache ID.
- size += 4;
-
- // Need 2 bytes for the number of partitions.
- size += 2;
-
- CacheState state = entry.getValue();
-
- // 2 bytes partition ID, size and counter per partition.
- size += 18 * state.size();
- }
-
- return size;
- }
-
- /**
- * @param buf Buffer to write to.
- * @param entry Data entry.
- */
- private void putDataEntry(ByteBuffer buf, DataEntry entry) throws IgniteCheckedException {
- buf.putInt(entry.cacheId());
-
- if (!entry.key().putValue(buf))
- throw new AssertionError();
-
- if (entry.value() == null)
- buf.putInt(-1);
- else if (!entry.value().putValue(buf))
- throw new AssertionError();
-
- buf.put((byte)entry.op().ordinal());
-
- putVersion(buf, entry.nearXidVersion(), true);
- putVersion(buf, entry.writeVersion(), false);
-
- buf.putInt(entry.partitionId());
- buf.putLong(entry.partitionCounter());
- buf.putLong(entry.expireTime());
- }
-
- /**
- * @param states Cache states.
- */
- private void putCacheStates(ByteBuffer buf, Map<Integer, CacheState> states) {
- buf.putShort((short)states.size());
-
- for (Map.Entry<Integer, CacheState> entry : states.entrySet()) {
- buf.putInt(entry.getKey());
-
- CacheState state = entry.getValue();
-
- // Need 2 bytes for the number of partitions.
- buf.putShort((short)state.size());
-
- for (int i = 0; i < state.size(); i++) {
- buf.putShort((short)state.partitionByIndex(i));
-
- buf.putLong(state.partitionSizeByIndex(i));
- buf.putLong(state.partitionCounterByIndex(i));
- }
- }
- }
-
- /**
- * @param in Input to read from.
- * @return Read entry.
- */
- private DataEntry readDataEntry(ByteBufferBackedDataInput in) throws IOException, IgniteCheckedException {
- int cacheId = in.readInt();
-
- int keySize = in.readInt();
- byte keyType = in.readByte();
- byte[] keyBytes = new byte[keySize];
- in.readFully(keyBytes);
-
- int valSize = in.readInt();
-
- byte valType = 0;
- byte[] valBytes = null;
-
- if (valSize >= 0) {
- valType = in.readByte();
- valBytes = new byte[valSize];
- in.readFully(valBytes);
- }
-
- byte ord = in.readByte();
-
- GridCacheOperation op = GridCacheOperation.fromOrdinal(ord & 0xFF);
-
- GridCacheVersion nearXidVer = readVersion(in, true);
- GridCacheVersion writeVer = readVersion(in, false);
-
- int partId = in.readInt();
- long partCntr = in.readLong();
- long expireTime = in.readLong();
-
- GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
-
- if (cacheCtx != null) {
- CacheObjectContext coCtx = cacheCtx.cacheObjectContext();
-
- KeyCacheObject key = co.toKeyCacheObject(coCtx, keyType, keyBytes);
- CacheObject val = valBytes != null ? co.toCacheObject(coCtx, valType, valBytes) : null;
-
- return new DataEntry(
- cacheId,
- key,
- val,
- op,
- nearXidVer,
- writeVer,
- expireTime,
- partId,
- partCntr
- );
- }
- else
- return new LazyDataEntry(
- cctx,
- cacheId,
- keyType,
- keyBytes,
- valType,
- valBytes,
- op,
- nearXidVer,
- writeVer,
- expireTime,
- partId,
- partCntr);
- }
-
- /**
- * @param buf Buffer to read from.
- * @return Read map.
- */
- private Map<Integer, CacheState> readPartitionStates(DataInput buf) throws IOException {
- int caches = buf.readShort() & 0xFFFF;
-
- if (caches == 0)
- return Collections.emptyMap();
-
- Map<Integer, CacheState> states = new HashMap<>(caches, 1.0f);
-
- for (int i = 0; i < caches; i++) {
- int cacheId = buf.readInt();
-
- int parts = buf.readShort() & 0xFFFF;
-
- CacheState state = new CacheState(parts);
-
- for (int p = 0; p < parts; p++) {
- int partId = buf.readShort() & 0xFFFF;
- long size = buf.readLong();
- long partCntr = buf.readLong();
-
- state.addPartitionState(partId, size, partCntr);
- }
-
- states.put(cacheId, state);
- }
-
- return states;
- }
-
- /**
- * @param buf Buffer.
- * @param ver Version to write.
- * @param allowNull Is {@code null}version allowed.
- */
- private void putVersion(ByteBuffer buf, GridCacheVersion ver, boolean allowNull) {
- CacheVersionIO.write(buf, ver, allowNull);
- }
-
- /**
- * Changes the buffer position by the number of read bytes.
- *
- * @param in Data input to read from.
- * @param allowNull Is {@code null}version allowed.
- * @return Read cache version.
- */
- private GridCacheVersion readVersion(ByteBufferBackedDataInput in, boolean allowNull) throws IOException {
- // To be able to read serialization protocol version.
- in.ensure(1);
-
- try {
- int size = CacheVersionIO.readSize(in.buffer(), allowNull);
-
- in.ensure(size);
-
- return CacheVersionIO.read(in.buffer(), allowNull);
- }
- catch (IgniteCheckedException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * @param buf Buffer.
- * @param rowBytes Row bytes.
- */
- @SuppressWarnings("unchecked")
- private static void putRow(ByteBuffer buf, byte[] rowBytes) {
- assert rowBytes.length > 0;
-
- buf.put(rowBytes);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/resources/META-INF/classnames.properties b/modules/pds/src/main/resources/META-INF/classnames.properties
deleted file mode 100644
index 3003e5f..0000000
--- a/modules/pds/src/main/resources/META-INF/classnames.properties
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-org.apache.ignite.internal.processors.cache.database.FullPageIdIterableComparator
-org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager$11
-org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager$7
-org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager$CheckpointEntryType
-org.apache.ignite.internal.processors.cache.database.GridCacheOffheapManager$RebalanceIteratorAdapter
-org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl$Segment
-org.apache.ignite.internal.processors.cache.database.wal.FileWriteAheadLogManager$FileArchiver$1
-org.apache.ignite.internal.processors.cache.database.wal.FileWriteAheadLogManager$Mode
-org.apache.ignite.internal.processors.cache.database.wal.FileWriteAheadLogManager$RecordsIterator
-org.apache.ignite.internal.processors.cache.database.wal.SegmentEofException
-org.apache.ignite.internal.processors.cache.database.wal.crc.IgniteDataIntegrityViolationException
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsAtomicCacheRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsAtomicCacheRebalancingTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsAtomicCacheRebalancingTest.java
deleted file mode 100644
index 0c4812d..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsAtomicCacheRebalancingTest.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-
-/**
- *
- */
-public class IgnitePdsAtomicCacheRebalancingTest extends IgnitePdsCacheRebalancingAbstractTest {
- /** {@inheritDoc} */
- @Override protected CacheConfiguration cacheConfiguration(String cacheName) {
- CacheConfiguration ccfg = new CacheConfiguration(cacheName);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
- ccfg.setCacheMode(CacheMode.PARTITIONED);
- ccfg.setBackups(1);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- return ccfg;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsCacheRebalancingAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsCacheRebalancingAbstractTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsCacheRebalancingAbstractTest.java
deleted file mode 100644
index 6c0ec19..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsCacheRebalancingAbstractTest.java
+++ /dev/null
@@ -1,523 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.Serializable;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.PartitionLossPolicy;
-import org.apache.ignite.cache.QueryEntity;
-import org.apache.ignite.cache.QueryIndex;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.Transaction;
-
-import static org.apache.ignite.testframework.GridTestUtils.runMultiThreadedAsync;
-
-/**
- * Test for rebalancing and persistence integration.
- */
-public abstract class IgnitePdsCacheRebalancingAbstractTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Cache name. */
- private final String cacheName = "cache";
-
- /** */
- protected boolean explicitTx;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration ccfg1 = cacheConfiguration(cacheName);
- ccfg1.setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
- ccfg1.setBackups(1);
- ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- CacheConfiguration ccfg2 = cacheConfiguration("indexed");
- ccfg2.setBackups(1);
- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- QueryEntity qryEntity = new QueryEntity(Integer.class.getName(), TestValue.class.getName());
-
- LinkedHashMap<String, String> fields = new LinkedHashMap<>();
-
- fields.put("v1", Integer.class.getName());
- fields.put("v2", Integer.class.getName());
-
- qryEntity.setFields(fields);
-
- QueryIndex qryIdx = new QueryIndex("v1", true);
-
- qryEntity.setIndexes(Collections.singleton(qryIdx));
-
- ccfg2.setQueryEntities(Collections.singleton(qryEntity));
-
- cfg.setCacheConfiguration(ccfg1, ccfg2);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- memCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
- memCfg.setPageSize(1024);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setSwapFilePath("work/swap");
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setDiscoverySpi(
- new TcpDiscoverySpi()
- .setIpFinder(IP_FINDER)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 20 * 60 * 1000;
- }
-
- /**
- * @param cacheName Cache name.
- * @return Cache configuration.
- */
- protected abstract CacheConfiguration cacheConfiguration(String cacheName);
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * Test that outdated partitions on restarted nodes are correctly replaced with newer versions.
- *
- * @throws Exception If fails.
- */
- public void testRebalancingOnRestart() throws Exception {
- Ignite ignite0 = startGrid(0);
-
- startGrid(1);
-
- IgniteEx ignite2 = startGrid(2);
-
- awaitPartitionMapExchange();
-
- IgniteCache<Integer, Integer> cache1 = ignite0.cache(cacheName);
-
- for (int i = 0; i < 5000; i++)
- cache1.put(i, i);
-
- ignite2.close();
-
- awaitPartitionMapExchange();
-
- ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
-
- assert cache1.lostPartitions().isEmpty();
-
- for (int i = 0; i < 5000; i++)
- cache1.put(i, i * 2);
-
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
-
- info(">>> Done puts...");
-
- ignite2 = startGrid(2);
-
- awaitPartitionMapExchange();
-
- IgniteCache<Integer, Integer> cache3 = ignite2.cache(cacheName);
-
- for (int i = 0; i < 100; i++)
- assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
- }
-
- /**
- * Test that outdated partitions on restarted nodes are correctly replaced with newer versions.
- *
- * @throws Exception If fails.
- */
- public void testRebalancingOnRestartAfterCheckpoint() throws Exception {
- IgniteEx ignite0 = startGrid(0);
-
- IgniteEx ignite1 = startGrid(1);
-
- IgniteEx ignite2 = startGrid(2);
- IgniteEx ignite3 = startGrid(3);
-
- ignite0.cache(cacheName).rebalance().get();
- ignite1.cache(cacheName).rebalance().get();
- ignite2.cache(cacheName).rebalance().get();
- ignite3.cache(cacheName).rebalance().get();
-
- awaitPartitionMapExchange();
-
- IgniteCache<Integer, Integer> cache1 = ignite0.cache(cacheName);
-
- for (int i = 0; i < 1000; i++)
- cache1.put(i, i);
-
- ignite0.context().cache().context().database().waitForCheckpoint("test");
- ignite1.context().cache().context().database().waitForCheckpoint("test");
-
- info("++++++++++ After checkpoint");
-
- ignite2.close();
- ignite3.close();
-
- awaitPartitionMapExchange();
-
- ignite0.resetLostPartitions(Collections.singletonList(cache1.getName()));
-
- assert cache1.lostPartitions().isEmpty();
-
- for (int i = 0; i < 1000; i++)
- cache1.put(i, i * 2);
-
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>");
-
- info(">>> Done puts...");
-
- ignite2 = startGrid(2);
- ignite3 = startGrid(3);
-
- ignite2.cache(cacheName).rebalance().get();
- ignite3.cache(cacheName).rebalance().get();
-
- awaitPartitionMapExchange();
-
- IgniteCache<Integer, Integer> cache2 = ignite2.cache(cacheName);
- IgniteCache<Integer, Integer> cache3 = ignite3.cache(cacheName);
-
- for (int i = 0; i < 100; i++) {
- assertEquals(String.valueOf(i), (Integer)(i * 2), cache2.get(i));
- assertEquals(String.valueOf(i), (Integer)(i * 2), cache3.get(i));
- }
- }
-
- /**
- * Test that all data is correctly restored after non-graceful restart.
- *
- * @throws Exception If fails.
- */
- public void testDataCorrectnessAfterRestart() throws Exception {
- IgniteEx ignite1 = (IgniteEx)G.start(getConfiguration("test1"));
- IgniteEx ignite2 = (IgniteEx)G.start(getConfiguration("test2"));
- IgniteEx ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
- IgniteEx ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
-
- awaitPartitionMapExchange();
-
- IgniteCache<Integer, Integer> cache1 = ignite1.cache(cacheName);
-
- for (int i = 0; i < 100; i++)
- cache1.put(i, i);
-
- ignite1.close();
- ignite2.close();
- ignite3.close();
- ignite4.close();
-
- ignite1 = (IgniteEx)G.start(getConfiguration("test1"));
- ignite2 = (IgniteEx)G.start(getConfiguration("test2"));
- ignite3 = (IgniteEx)G.start(getConfiguration("test3"));
- ignite4 = (IgniteEx)G.start(getConfiguration("test4"));
-
- awaitPartitionMapExchange();
-
- cache1 = ignite1.cache(cacheName);
- IgniteCache<Integer, Integer> cache2 = ignite2.cache(cacheName);
- IgniteCache<Integer, Integer> cache3 = ignite3.cache(cacheName);
- IgniteCache<Integer, Integer> cache4 = ignite4.cache(cacheName);
-
- for (int i = 0; i < 100; i++) {
- assert cache1.get(i).equals(i);
- assert cache2.get(i).equals(i);
- assert cache3.get(i).equals(i);
- assert cache4.get(i).equals(i);
- }
- }
-
- /**
- * Test that partitions are marked as lost when all owners leave cluster, but recover after nodes rejoin.
- *
- * @throws Exception If fails.
- */
- public void testPartitionLossAndRecover() throws Exception {
- fail("IGNITE-5302");
-
- Ignite ignite1 = startGrid(0);
- Ignite ignite2 = startGrid(1);
- Ignite ignite3 = startGrid(2);
- Ignite ignite4 = startGrid(3);
-
- awaitPartitionMapExchange();
-
- IgniteCache<String, String> cache1 = ignite1.cache(cacheName);
-
- final int offset = 10;
-
- for (int i = 0; i < 100; i++)
- cache1.put(String.valueOf(i), String.valueOf(i + offset));
-
- ignite3.close();
- ignite4.close();
-
- awaitPartitionMapExchange();
-
- assert !ignite1.cache(cacheName).lostPartitions().isEmpty();
-
- ignite3 = startGrid(2);
- ignite4 = startGrid(3);
-
- ignite1.resetLostPartitions(Collections.singletonList(cacheName));
-
- IgniteCache<String, String> cache2 = ignite2.cache(cacheName);
- IgniteCache<String, String> cache3 = ignite3.cache(cacheName);
- IgniteCache<String, String> cache4 = ignite4.cache(cacheName);
-
- //Thread.sleep(5_000);
-
- for (int i = 0; i < 100; i++) {
- String key = String.valueOf(i);
- String expected = String.valueOf(i + offset);
-
- assertEquals(expected, cache1.get(key));
- assertEquals(expected, cache2.get(key));
- assertEquals(expected, cache3.get(key));
- assertEquals(expected, cache4.get(key));
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTopologyChangesWithConstantLoad() throws Exception {
- final int entriesCnt = 10_000;
- int maxNodesCount = 4;
- int topChanges = 20;
- final String cacheName = "indexed";
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- final ConcurrentMap<Integer, TestValue> map = new ConcurrentHashMap<>();
-
- Ignite ignite = startGrid(0);
-
- IgniteCache<Integer, TestValue> cache = ignite.cache(cacheName);
-
- for (int i = 0; i < entriesCnt; i++) {
- cache.put(i, new TestValue(i, i));
- map.put(i, new TestValue(i, i));
- }
-
- final AtomicInteger nodesCnt = new AtomicInteger();
-
- IgniteInternalFuture fut = runMultiThreadedAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (true) {
- if (stop.get())
- return null;
-
- int k = ThreadLocalRandom.current().nextInt(entriesCnt);
- int v1 = ThreadLocalRandom.current().nextInt();
- int v2 = ThreadLocalRandom.current().nextInt();
-
- int n = nodesCnt.get();
-
- if (n <= 0)
- continue;
-
- Ignite ignite;
-
- try {
- ignite = grid(ThreadLocalRandom.current().nextInt(n));
- }
- catch (Exception ignored) {
- continue;
- }
-
- if (ignite == null)
- continue;
-
- Transaction tx = null;
- boolean success = true;
-
- if (explicitTx)
- tx = ignite.transactions().txStart();
-
- try {
- ignite.cache(cacheName).put(k, new TestValue(v1, v2));
- }
- catch (Exception ignored) {
- success = false;
- }
- finally {
- if (tx != null) {
- try {
- tx.commit();
- }
- catch (Exception ignored) {
- success = false;
- }
- }
- }
-
- if (success)
- map.put(k, new TestValue(v1, v2));
- }
- }
- }, 1, "load-runner");
-
- for (int i = 0; i < topChanges; i++) {
- U.sleep(3_000);
-
- boolean add;
-
- if (nodesCnt.get() <= maxNodesCount / 2)
- add = true;
- else if (nodesCnt.get() > maxNodesCount)
- add = false;
- else
- add = ThreadLocalRandom.current().nextBoolean();
-
- if (add)
- startGrid(nodesCnt.incrementAndGet());
- else
- stopGrid(nodesCnt.getAndDecrement());
-
- awaitPartitionMapExchange();
-
- cache.rebalance().get();
- }
-
- stop.set(true);
-
- fut.get();
-
- awaitPartitionMapExchange();
-
- for (Map.Entry<Integer, TestValue> entry : map.entrySet())
- assertEquals(Integer.toString(entry.getKey()), entry.getValue(), cache.get(entry.getKey()));
- }
-
- /**
- *
- */
- private static class TestValue implements Serializable {
- /** V 1. */
- private final int v1;
- /** V 2. */
- private final int v2;
-
- /**
- * @param v1 V 1.
- * @param v2 V 2.
- */
- private TestValue(int v1, int v2) {
- this.v1 = v1;
- this.v2 = v2;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
-
- TestValue val = (TestValue)o;
-
- return v1 == val.v1 && v2 == val.v2;
-
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = v1;
-
- res = 31 * res + v2;
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "TestValue{" +
- "v1=" + v1 +
- ", v2=" + v2 +
- '}';
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsClientNearCachePutGetTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsClientNearCachePutGetTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsClientNearCachePutGetTest.java
deleted file mode 100644
index 240171e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsClientNearCachePutGetTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class IgnitePdsClientNearCachePutGetTest extends IgniteDbClientNearCachePutGetTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
[10/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/SegmentEofException.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/SegmentEofException.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/SegmentEofException.java
deleted file mode 100644
index fcb9fe3..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/SegmentEofException.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-import org.apache.ignite.IgniteCheckedException;
-
-/**
- * This exception is thrown either when we reach the end of file of WAL segment, or when we encounter
- * a record with type equal to {@code 0}.
- */
-public class SegmentEofException extends IgniteCheckedException {
- /** */
- private static final long serialVersionUID = 0L;
-
- /**
- * @param msg Message.
- * @param cause Cause.
- */
- public SegmentEofException(String msg, Throwable cause) {
- super(msg, cause, false);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/IgniteDataIntegrityViolationException.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/IgniteDataIntegrityViolationException.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/IgniteDataIntegrityViolationException.java
deleted file mode 100644
index 7c3eaa2..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/IgniteDataIntegrityViolationException.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal.crc;
-
-import org.apache.ignite.IgniteException;
-
-/**
- * Will be thrown if data integrity violation is found
- */
-public class IgniteDataIntegrityViolationException extends IgniteException {
- /** */
- private static final long serialVersionUID = 0L;
-
- /**
- * Default constructor.
- */
- public IgniteDataIntegrityViolationException() {
- }
-
- /**
- * @param msg Message.
- */
- public IgniteDataIntegrityViolationException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/PureJavaCrc32.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/PureJavaCrc32.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/PureJavaCrc32.java
deleted file mode 100644
index a947b28..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/crc/PureJavaCrc32.java
+++ /dev/null
@@ -1,655 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal.crc;
-
-
-import java.nio.ByteBuffer;
-
-/**
- * This class was taken from Hadoop org.apache.hadoop.util.PureJavaCrc32
- *
- * A pure-java implementation of the CRC32 checksum that uses the same polynomial as the built-in native CRC32.
- *
- * This is to avoid the JNI overhead for certain uses of Checksumming where many small pieces of data are checksummed in
- * succession.
- *
- * The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6
- */
-public class PureJavaCrc32 {
- /**
- * the current CRC value
- */
- private int crc;
-
- /**
- * Create a new PureJavaCrc32 object.
- */
- public PureJavaCrc32() {
- reset();
- }
-
- /**
- * return crc32 value
- */
- public int getValue() {
- return crc;
- }
-
- /**
- * preparation for further calculations
- */
- public void reset() {
- crc = 0xffffffff;
- }
-
- /**
- * @param b B.
- * @param len Length.
- */
- public void update(final ByteBuffer b, final int len) {
- crc = calcCrc32(crc, b, len);
- }
-
- /**
- * @param b B.
- */
- final public void update(int b) {
- crc = (crc >>> 8) ^ T[(((crc ^ b) << 24) >>> 24)];
- }
-
-
- /**
- * Calculates CRC32 checksum. This method will move buffer's position to {@code len} bytes forward.
- *
- * @param b B.
- * @param len Length.
- */
- public static int calcCrc32(ByteBuffer b, final int len) {
- return calcCrc32(0xffffffff, b, len);
- }
-
- /**
- * @param crc0 Crc 0.
- * @param b B.
- * @param len Length.
- */
- private static int calcCrc32(int crc0, ByteBuffer b, final int len) {
- int crc = crc0;
-
- final int remainder = len & 0x7;
- int i = 0;
- for (; i < len - remainder; i += 8) {
- final int x = crc
- ^ ((((b.get() << 24) >>> 24) + ((b.get() << 24) >>> 16))
- + (((b.get() << 24) >>> 8) + (b.get() << 24)));
-
- crc = ((T[((x << 24) >>> 24) + 0x700] ^ T[((x << 16) >>> 24) + 0x600])
- ^ (T[((x << 8) >>> 24) + 0x500] ^ T[(x >>> 24) + 0x400]))
- ^ ((T[((b.get() << 24) >>> 24) + 0x300] ^ T[((b.get() << 24) >>> 24) + 0x200])
- ^ (T[((b.get() << 24) >>> 24) + 0x100] ^ T[((b.get() << 24) >>> 24)]));
- }
-
- /* loop unroll - duff's device style */
- switch (remainder) {
- case 7:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- case 6:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- case 5:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- case 4:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- case 3:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- case 2:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- case 1:
- crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
- default:
- /* nothing */
- }
-
- return crc;
- }
-
- /**
- * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
- */
- private static final int[] T = new int[]{
- /* T8_0 */
- 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
- 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
- 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
- 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
- 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
- 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
- 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
- 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
- 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
- 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
- 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940,
- 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
- 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
- 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
- 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
- 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
- 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
- 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
- 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
- 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
- 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
- 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
- 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
- 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
- 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
- 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
- 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
- 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
- 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
- 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
- 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
- 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
- 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
- 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
- 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
- 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
- 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
- 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
- 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
- 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
- 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
- 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
- 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
- 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
- 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
- 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
- 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04,
- 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
- 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
- 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
- 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
- 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
- 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
- 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
- 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
- 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
- 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
- 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
- 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
- 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
- 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
- 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
- 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
- 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
- /* T8_1 */
- 0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3,
- 0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7,
- 0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB,
- 0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF,
- 0x4AC21251, 0x53D92310, 0x78F470D3, 0x61EF4192,
- 0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496,
- 0x821B9859, 0x9B00A918, 0xB02DFADB, 0xA936CB9A,
- 0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E,
- 0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761,
- 0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265,
- 0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69,
- 0x39316BAE, 0x202A5AEF, 0x0B07092C, 0x121C386D,
- 0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530,
- 0xBB2AF3F7, 0xA231C2B6, 0x891C9175, 0x9007A034,
- 0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38,
- 0x73F379FF, 0x6AE848BE, 0x41C51B7D, 0x58DE2A3C,
- 0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6,
- 0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2,
- 0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE,
- 0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA,
- 0xBABB5D54, 0xA3A06C15, 0x888D3FD6, 0x91960E97,
- 0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93,
- 0x7262D75C, 0x6B79E61D, 0x4054B5DE, 0x594F849F,
- 0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B,
- 0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, 0x4ED03864,
- 0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60,
- 0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C,
- 0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768,
- 0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35,
- 0x4B53BCF2, 0x52488DB3, 0x7965DE70, 0x607EEF31,
- 0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D,
- 0x838A36FA, 0x9A9107BB, 0xB1BC5478, 0xA8A76539,
- 0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88,
- 0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, 0x74C20E8C,
- 0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180,
- 0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484,
- 0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9,
- 0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD,
- 0xB9980012, 0xA0833153, 0x8BAE6290, 0x92B553D1,
- 0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5,
- 0xAE07BCE9, 0xB71C8DA8, 0x9C31DE6B, 0x852AEF2A,
- 0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E,
- 0x66DE36E1, 0x7FC507A0, 0x54E85463, 0x4DF36522,
- 0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026,
- 0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B,
- 0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F,
- 0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773,
- 0x4870E1B4, 0x516BD0F5, 0x7A468336, 0x635DB277,
- 0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D,
- 0xAF96124A, 0xB68D230B, 0x9DA070C8, 0x84BB4189,
- 0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85,
- 0x674F9842, 0x7E54A903, 0x5579FAC0, 0x4C62CB81,
- 0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC,
- 0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8,
- 0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4,
- 0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0,
- 0x5E7EF3EC, 0x4765C2AD, 0x6C48916E, 0x7553A02F,
- 0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B,
- 0x96A779E4, 0x8FBC48A5, 0xA4911B66, 0xBD8A2A27,
- 0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23,
- 0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E,
- 0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A,
- 0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876,
- 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
- /* T8_2 */
- 0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59,
- 0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685,
- 0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1,
- 0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D,
- 0x1C26A370, 0x1DE4C947, 0x1FA2771E, 0x1E601D29,
- 0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5,
- 0x1235F2C8, 0x13F798FF, 0x11B126A6, 0x10734C91,
- 0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D,
- 0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9,
- 0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065,
- 0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901,
- 0x3157BF84, 0x3095D5B3, 0x32D36BEA, 0x331101DD,
- 0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9,
- 0x23624D4C, 0x22A0277B, 0x20E69922, 0x2124F315,
- 0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71,
- 0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, 0x2F37A2AD,
- 0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399,
- 0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45,
- 0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221,
- 0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD,
- 0x6CBC2EB0, 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9,
- 0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835,
- 0x62AF7F08, 0x636D153F, 0x612BAB66, 0x60E9C151,
- 0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D,
- 0x48D7CB20, 0x4915A117, 0x4B531F4E, 0x4A917579,
- 0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5,
- 0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1,
- 0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D,
- 0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609,
- 0x53F8C08C, 0x523AAABB, 0x507C14E2, 0x51BE7ED5,
- 0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1,
- 0x5DEB9134, 0x5C29FB03, 0x5E6F455A, 0x5FAD2F6D,
- 0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9,
- 0xE63CB35C, 0xE7FED96B, 0xE5B86732, 0xE47A0D05,
- 0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461,
- 0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD,
- 0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9,
- 0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75,
- 0xF300E948, 0xF2C2837F, 0xF0843D26, 0xF1465711,
- 0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD,
- 0xD9785D60, 0xD8BA3757, 0xDAFC890E, 0xDB3EE339,
- 0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5,
- 0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, 0xD52DB281,
- 0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D,
- 0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049,
- 0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895,
- 0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1,
- 0xCC440774, 0xCD866D43, 0xCFC0D31A, 0xCE02B92D,
- 0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819,
- 0x96A63E9C, 0x976454AB, 0x9522EAF2, 0x94E080C5,
- 0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1,
- 0x98B56F24, 0x99770513, 0x9B31BB4A, 0x9AF3D17D,
- 0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69,
- 0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5,
- 0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1,
- 0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D,
- 0xA9E2D0A0, 0xA820BA97, 0xAA6604CE, 0xABA46EF9,
- 0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625,
- 0xA7F18118, 0xA633EB2F, 0xA4755576, 0xA5B73F41,
- 0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D,
- 0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89,
- 0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555,
- 0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31,
- 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
- /* T8_3 */
- 0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE,
- 0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9,
- 0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701,
- 0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056,
- 0x5019579F, 0xE8A530FA, 0xFA109F14, 0x42ACF871,
- 0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26,
- 0x95AD7F70, 0x2D111815, 0x3FA4B7FB, 0x8718D09E,
- 0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9,
- 0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0,
- 0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787,
- 0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F,
- 0xEAE41086, 0x525877E3, 0x40EDD80D, 0xF851BF68,
- 0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F,
- 0x7F496FF6, 0xC7F50893, 0xD540A77D, 0x6DFCC018,
- 0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0,
- 0xBAFD4719, 0x0241207C, 0x10F48F92, 0xA848E8F7,
- 0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3,
- 0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084,
- 0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C,
- 0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B,
- 0xCB0D0FA2, 0x73B168C7, 0x6104C729, 0xD9B8A04C,
- 0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B,
- 0x0EB9274D, 0xB6054028, 0xA4B0EFC6, 0x1C0C88A3,
- 0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4,
- 0x3B26F703, 0x839A9066, 0x912F3F88, 0x299358ED,
- 0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA,
- 0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002,
- 0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755,
- 0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72,
- 0xE45D37CB, 0x5CE150AE, 0x4E54FF40, 0xF6E89825,
- 0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D,
- 0x21E91F24, 0x99557841, 0x8BE0D7AF, 0x335CB0CA,
- 0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5,
- 0x623B216C, 0xDA874609, 0xC832E9E7, 0x708E8E82,
- 0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A,
- 0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D,
- 0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A,
- 0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D,
- 0x78F4C94B, 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5,
- 0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2,
- 0x4D6B1905, 0xF5D77E60, 0xE762D18E, 0x5FDEB6EB,
- 0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC,
- 0x88DF31EA, 0x3063568F, 0x22D6F961, 0x9A6A9E04,
- 0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953,
- 0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174,
- 0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623,
- 0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B,
- 0x57A4F122, 0xEF189647, 0xFDAD39A9, 0x45115ECC,
- 0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8,
- 0xF92F7951, 0x41931E34, 0x5326B1DA, 0xEB9AD6BF,
- 0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907,
- 0x3C9B51BE, 0x842736DB, 0x96929935, 0x2E2EFE50,
- 0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677,
- 0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120,
- 0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98,
- 0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF,
- 0xD67F4138, 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6,
- 0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981,
- 0x13CB69D7, 0xAB770EB2, 0xB9C2A15C, 0x017EC639,
- 0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E,
- 0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949,
- 0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E,
- 0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6,
- 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
- /* T8_4 */
- 0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0,
- 0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10,
- 0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111,
- 0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1,
- 0x60E09782, 0x5D80BE32, 0x1A20C4E2, 0x2740ED52,
- 0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92,
- 0x5090DC43, 0x6DF0F5F3, 0x2A508F23, 0x1730A693,
- 0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053,
- 0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4,
- 0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314,
- 0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15,
- 0x0431C205, 0x3951EBB5, 0x7EF19165, 0x4391B8D5,
- 0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256,
- 0x54A11E46, 0x69C137F6, 0x2E614D26, 0x13016496,
- 0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997,
- 0x64D15587, 0x59B17C37, 0x1E1106E7, 0x23712F57,
- 0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299,
- 0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459,
- 0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958,
- 0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98,
- 0x3813CFCB, 0x0573E67B, 0x42D39CAB, 0x7FB3B51B,
- 0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB,
- 0x0863840A, 0x3503ADBA, 0x72A3D76A, 0x4FC3FEDA,
- 0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A,
- 0x9932774D, 0xA4525EFD, 0xE3F2242D, 0xDE920D9D,
- 0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D,
- 0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C,
- 0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C,
- 0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F,
- 0x0C52460F, 0x31326FBF, 0x7692156F, 0x4BF23CDF,
- 0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE,
- 0x3C220DCE, 0x0142247E, 0x46E25EAE, 0x7B82771E,
- 0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42,
- 0x44661652, 0x79063FE2, 0x3EA64532, 0x03C66C82,
- 0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183,
- 0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743,
- 0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0,
- 0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00,
- 0xE1766CD1, 0xDC164561, 0x9BB63FB1, 0xA6D61601,
- 0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1,
- 0x70279F96, 0x4D47B626, 0x0AE7CCF6, 0x3787E546,
- 0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386,
- 0x4057D457, 0x7D37FDE7, 0x3A978737, 0x07F7AE87,
- 0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847,
- 0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4,
- 0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404,
- 0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905,
- 0xD537E515, 0xE857CCA5, 0xAFF7B675, 0x92979FC5,
- 0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B,
- 0x1C954E1B, 0x21F567AB, 0x66551D7B, 0x5B3534CB,
- 0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA,
- 0x2CE505DA, 0x11852C6A, 0x562556BA, 0x6B457F0A,
- 0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589,
- 0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349,
- 0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48,
- 0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888,
- 0x28D4C7DF, 0x15B4EE6F, 0x521494BF, 0x6F74BD0F,
- 0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF,
- 0x18A48C1E, 0x25C4A5AE, 0x6264DF7E, 0x5F04F6CE,
- 0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E,
- 0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D,
- 0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D,
- 0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C,
- 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
- /* T8_5 */
- 0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE,
- 0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8,
- 0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3,
- 0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5,
- 0x03D6029B, 0xC88AD13E, 0x4E1EA390, 0x85427035,
- 0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223,
- 0xEF8580F6, 0x24D95353, 0xA24D21FD, 0x6911F258,
- 0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E,
- 0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798,
- 0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E,
- 0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5,
- 0x706EC54D, 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3,
- 0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503,
- 0x9FEB45BB, 0x54B7961E, 0xD223E4B0, 0x197F3715,
- 0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E,
- 0x73B8C7D6, 0xB8E41473, 0x3E7066DD, 0xF52CB578,
- 0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2,
- 0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4,
- 0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF,
- 0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9,
- 0x0C8E08F7, 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59,
- 0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F,
- 0xE0DD8A9A, 0x2B81593F, 0xAD152B91, 0x6649F834,
- 0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22,
- 0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, 0x8E607DF4,
- 0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2,
- 0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99,
- 0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F,
- 0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F,
- 0x90B34FD7, 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79,
- 0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02,
- 0x7CE0CDBA, 0xB7BC1E1F, 0x31286CB1, 0xFA74BF14,
- 0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676,
- 0x852156CE, 0x4E7D856B, 0xC8E9F7C5, 0x03B52460,
- 0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B,
- 0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D,
- 0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED,
- 0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB,
- 0xF135942E, 0x3A69478B, 0xBCFD3525, 0x77A1E680,
- 0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496,
- 0x191C11EE, 0xD240C24B, 0x54D4B0E5, 0x9F886340,
- 0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156,
- 0xF54F9383, 0x3E134026, 0xB8873288, 0x73DBE12D,
- 0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B,
- 0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB,
- 0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD,
- 0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6,
- 0x6D08D30E, 0xA65400AB, 0x20C07205, 0xEB9CA1A0,
- 0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A,
- 0x8A795CA2, 0x41258F07, 0xC7B1FDA9, 0x0CED2E0C,
- 0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77,
- 0x662ADECF, 0xAD760D6A, 0x2BE27FC4, 0xE0BEAC61,
- 0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81,
- 0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97,
- 0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC,
- 0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA,
- 0x16441B82, 0xDD18C827, 0x5B8CBA89, 0x90D0692C,
- 0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A,
- 0xFA1799EF, 0x314B4A4A, 0xB7DF38E4, 0x7C83EB41,
- 0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957,
- 0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7,
- 0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1,
- 0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA,
- 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
- /* T8_6 */
- 0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D,
- 0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E,
- 0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA,
- 0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9,
- 0x67DE9CCE, 0xC1A9977A, 0xF0418DE7, 0x56368653,
- 0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240,
- 0x5431D2A9, 0xF246D91D, 0xC3AEC380, 0x65D9C834,
- 0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27,
- 0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301,
- 0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712,
- 0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66,
- 0x081D53E8, 0xAE6A585C, 0x9F8242C1, 0x39F54975,
- 0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF,
- 0x5C2C8141, 0xFA5B8AF5, 0xCBB39068, 0x6DC49BDC,
- 0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8,
- 0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, 0x5E2BD5BB,
- 0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4,
- 0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7,
- 0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183,
- 0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590,
- 0x23D5E9B7, 0x85A2E203, 0xB44AF89E, 0x123DF32A,
- 0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739,
- 0x103AA7D0, 0xB64DAC64, 0x87A5B6F9, 0x21D2BD4D,
- 0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E,
- 0x8BB64CE5, 0x2DC14751, 0x1C295DCC, 0xBA5E5678,
- 0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B,
- 0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F,
- 0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C,
- 0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6,
- 0x1827F438, 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5,
- 0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1,
- 0x2BC8BA5F, 0x8DBFB1EB, 0xBC57AB76, 0x1A20A0C2,
- 0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F,
- 0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, 0x4DB1D47C,
- 0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08,
- 0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B,
- 0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1,
- 0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2,
- 0xDC27385B, 0x7A5033EF, 0x4BB82972, 0xEDCF22C6,
- 0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5,
- 0x47ABD36E, 0xE1DCD8DA, 0xD034C247, 0x7643C9F3,
- 0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0,
- 0x74449D09, 0xD23396BD, 0xE3DB8C20, 0x45AC8794,
- 0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387,
- 0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D,
- 0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E,
- 0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A,
- 0xE7D525D4, 0x41A22E60, 0x704A34FD, 0xD63D3F49,
- 0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516,
- 0x3852BB98, 0x9E25B02C, 0xAFCDAAB1, 0x09BAA105,
- 0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71,
- 0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, 0x3A55EF62,
- 0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8,
- 0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB,
- 0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF,
- 0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC,
- 0x03A0A617, 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A,
- 0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899,
- 0x304FE870, 0x9638E3C4, 0xA7D0F959, 0x01A7F2ED,
- 0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE,
- 0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044,
- 0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457,
- 0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23,
- 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
- /* T8_7 */
- 0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3,
- 0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919,
- 0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56,
- 0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC,
- 0x7CBB312B, 0xB01131B5, 0x3E9E3656, 0xF23436C8,
- 0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832,
- 0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, 0x21D12D7D,
- 0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387,
- 0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5,
- 0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F,
- 0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00,
- 0xAED97719, 0x62737787, 0xECFC7064, 0x205670FA,
- 0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E,
- 0x01875D87, 0xCD2D5D19, 0x43A25AFA, 0x8F085A64,
- 0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B,
- 0xD2624632, 0x1EC846AC, 0x9047414F, 0x5CED41D1,
- 0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E,
- 0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4,
- 0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB,
- 0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041,
- 0x5526F3C6, 0x998CF358, 0x1703F4BB, 0xDBA9F425,
- 0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF,
- 0x86C3E873, 0x4A69E8ED, 0xC4E6EF0E, 0x084CEF90,
- 0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A,
- 0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, 0x5E64A758,
- 0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2,
- 0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED,
- 0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217,
- 0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673,
- 0x281A9F6A, 0xE4B09FF4, 0x6A3F9817, 0xA6959889,
- 0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6,
- 0xFBFF84DF, 0x37558441, 0xB9DA83A2, 0x7570833C,
- 0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239,
- 0xD7718B20, 0x1BDB8BBE, 0x95548C5D, 0x59FE8CC3,
- 0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C,
- 0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776,
- 0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312,
- 0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8,
- 0xFC65AF44, 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7,
- 0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D,
- 0xAA4DE78C, 0x66E7E712, 0xE868E0F1, 0x24C2E06F,
- 0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95,
- 0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, 0xF727FBDA,
- 0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520,
- 0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144,
- 0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE,
- 0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1,
- 0x8159C3E8, 0x4DF3C376, 0xC37CC495, 0x0FD6C40B,
- 0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4,
- 0xFEEC49CD, 0x32464953, 0xBCC94EB0, 0x70634E2E,
- 0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61,
- 0x2D095278, 0xE1A352E6, 0x6F2C5505, 0xA386559B,
- 0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF,
- 0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05,
- 0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A,
- 0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0,
- 0x83D02561, 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282,
- 0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78,
- 0x50353ED4, 0x9C9F3E4A, 0x121039A9, 0xDEBA3937,
- 0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD,
- 0xFF6B144A, 0x33C114D4, 0xBD4E1337, 0x71E413A9,
- 0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53,
- 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C,
- 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6
- };
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/record/HeaderRecord.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/record/HeaderRecord.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/record/HeaderRecord.java
deleted file mode 100644
index 35ce761..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/record/HeaderRecord.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal.record;
-
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-
-/**
- * Header record.
- */
-public class HeaderRecord extends WALRecord {
- /** */
- public static final long MAGIC = 0xB0D045A_CE7ED045AL;
-
- /** Serializer version */
- private final int ver;
-
- /**
- * @param ver Version.
- */
- public HeaderRecord(int ver) {
- this.ver = ver;
- }
-
- /**
- * @return Version.
- */
- public int version() {
- return ver;
- }
-
- /** {@inheritDoc} */
- @Override public RecordType type() {
- return RecordType.HEADER_RECORD;
- }
-}
[11/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWriteAheadLogManager.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWriteAheadLogManager.java
deleted file mode 100644
index 51a2437..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/FileWriteAheadLogManager.java
+++ /dev/null
@@ -1,2533 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileFilter;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.LockSupport;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.regex.Pattern;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.PersistenceMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.wal.record.HeaderRecord;
-import org.apache.ignite.internal.processors.cache.database.wal.serializer.RecordV1Serializer;
-import org.apache.ignite.internal.util.GridCloseableIteratorAdapter;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * File WAL manager.
- */
-public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter implements IgniteWriteAheadLogManager {
- /** */
- public static final FileDescriptor[] EMPTY_DESCRIPTORS = new FileDescriptor[0];
-
- /** */
- public static final String WAL_SEGMENT_FILE_EXT = ".wal";
-
- /** */
- private static final byte[] FILL_BUF = new byte[1024 * 1024];
-
- /** */
- private static final Pattern WAL_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal");
-
- /** */
- private static final Pattern WAL_TEMP_NAME_PATTERN = Pattern.compile("\\d{16}\\.wal\\.tmp");
-
- /** */
- private static final FileFilter WAL_SEGMENT_FILE_FILTER = new FileFilter() {
- @Override public boolean accept(File file) {
- return !file.isDirectory() && WAL_NAME_PATTERN.matcher(file.getName()).matches();
- }
- };
-
- /** */
- private static final FileFilter WAL_SEGMENT_TEMP_FILE_FILTER = new FileFilter() {
- @Override public boolean accept(File file) {
- return !file.isDirectory() && WAL_TEMP_NAME_PATTERN.matcher(file.getName()).matches();
- }
- };
-
- /** */
- private final boolean alwaysWriteFullPages;
-
- /** WAL segment size in bytes */
- private final long maxWalSegmentSize;
-
- /** */
- private final WALMode mode;
-
- /** Thread local byte buffer size, see {@link #tlb} */
- private final int tlbSize;
-
- /** WAL flush frequency. Makes sense only for {@link WALMode#BACKGROUND} log WALMode. */
- public final int flushFreq;
-
- /** Fsync delay. */
- private final long fsyncDelay;
-
- /** */
- private final PersistentStoreConfiguration psCfg;
-
- /** */
- private IgniteConfiguration igCfg;
-
- /** Persistence metrics tracker. */
- private PersistenceMetricsImpl metrics;
-
- /** */
- private File walWorkDir;
-
- /** */
- private File walArchiveDir;
-
- /** */
- private RecordSerializer serializer;
-
- /** */
- private volatile long oldestArchiveSegmentIdx;
-
- /** Updater for {@link #currentHnd}, used for verify there are no concurrent update for current log segment handle */
- private static final AtomicReferenceFieldUpdater<FileWriteAheadLogManager, FileWriteHandle> currentHndUpd =
- AtomicReferenceFieldUpdater.newUpdater(FileWriteAheadLogManager.class, FileWriteHandle.class, "currentHnd");
-
- /**
- * Thread local byte buffer for saving serialized WAL records chain, see {@link FileWriteHandle#head}.
- * Introduced to decrease number of buffers allocation.
- * Used only for record itself is shorter than {@link #tlbSize}.
- */
- private final ThreadLocal<ByteBuffer> tlb = new ThreadLocal<ByteBuffer>() {
- @Override protected ByteBuffer initialValue() {
- ByteBuffer buf = ByteBuffer.allocateDirect(tlbSize);
-
- buf.order(GridUnsafe.NATIVE_BYTE_ORDER);
-
- return buf;
- }
- };
-
- /** */
- private volatile FileArchiver archiver;
-
- /** */
- private QueueFlusher flusher;
-
- /** */
- private final ThreadLocal<WALPointer> lastWALPtr = new ThreadLocal<>();
-
- /** Current log segment handle */
- private volatile FileWriteHandle currentHnd;
-
- /**
- * @param ctx Kernal context.
- */
- public FileWriteAheadLogManager(GridKernalContext ctx) {
- igCfg = ctx.config();
-
- PersistentStoreConfiguration psCfg = igCfg.getPersistentStoreConfiguration();
-
- assert psCfg != null : "WAL should not be created if persistence is disabled.";
-
- this.psCfg = psCfg;
-
- maxWalSegmentSize = psCfg.getWalSegmentSize();
- mode = psCfg.getWalMode();
- tlbSize = psCfg.getTlbSize();
- flushFreq = psCfg.getWalFlushFrequency();
- fsyncDelay = psCfg.getWalFsyncDelay();
- alwaysWriteFullPages = psCfg.isAlwaysWriteFullPages();
- }
-
- /** {@inheritDoc} */
- @Override public void start0() throws IgniteCheckedException {
- if (!cctx.kernalContext().clientNode()) {
- String consId = consistentId();
-
- A.notNullOrEmpty(consId, "consistentId");
-
- consId = U.maskForFileName(consId);
-
- checkWalConfiguration();
-
- walWorkDir = initDirectory(
- psCfg.getWalStorePath(),
- PersistentStoreConfiguration.DFLT_WAL_STORE_PATH,
- consId,
- "write ahead log work directory"
- );
-
- walArchiveDir = initDirectory(
- psCfg.getWalArchivePath(),
- PersistentStoreConfiguration.DFLT_WAL_ARCHIVE_PATH,
- consId,
- "write ahead log archive directory"
- );
-
- serializer = new RecordV1Serializer(cctx);
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)cctx.database();
-
- metrics = dbMgr.persistentStoreMetricsImpl();
-
- checkOrPrepareFiles();
-
- IgniteBiTuple<Long, Long> tup = scanMinMaxArchiveIndices();
-
- oldestArchiveSegmentIdx = tup == null ? 0 : tup.get1();
-
- archiver = new FileArchiver(tup == null ? -1 : tup.get2());
-
- if (mode != WALMode.DEFAULT) {
- if (log.isInfoEnabled())
- log.info("Started write-ahead log manager [mode=" + mode + ']');
- }
- }
- }
-
- /**
- * @throws IgniteCheckedException if WAL store path is configured and archive path isn't (or vice versa)
- */
- private void checkWalConfiguration() throws IgniteCheckedException {
- if (psCfg.getWalStorePath() == null ^ psCfg.getWalArchivePath() == null) {
- throw new IgniteCheckedException(
- "Properties should be either both specified or both null " +
- "[walStorePath = " + psCfg.getWalStorePath() +
- ", walArchivePath = " + psCfg.getWalArchivePath() + "]"
- );
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException {
- super.onKernalStart0(reconnect);
-
- if (!cctx.kernalContext().clientNode() && cctx.kernalContext().state().active())
- archiver.start();
- }
-
- /**
- * @return Consistent ID.
- */
- protected String consistentId() {
- return cctx.discovery().consistentId().toString();
- }
-
- /** {@inheritDoc} */
- @Override protected void stop0(boolean cancel) {
- FileWriteHandle currentHnd = currentHandle();
-
- try {
- QueueFlusher flusher0 = flusher;
-
- if (flusher0 != null) {
- flusher0.shutdown();
-
- if (currentHnd != null)
- currentHnd.flush((FileWALPointer)null);
- }
-
- if (currentHnd != null)
- currentHnd.close(false);
-
- if (archiver != null)
- archiver.shutdown();
- }
- catch (Exception e) {
- U.error(log, "Failed to gracefully close WAL segment: " + currentHnd.file, e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Activated file write ahead log manager [nodeId=" + cctx.localNodeId() +
- " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
-
- start0();
-
- if (!cctx.kernalContext().clientNode()) {
- assert archiver != null;
-
- archiver.start();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onDeActivate(GridKernalContext kctx) {
- if (log.isDebugEnabled())
- log.debug("DeActivate file write ahead log [nodeId=" + cctx.localNodeId() +
- " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
-
- stop0(true);
-
- currentHnd = null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isAlwaysWriteFullPages() {
- return alwaysWriteFullPages;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isFullSync() {
- return mode == WALMode.DEFAULT;
- }
-
- /** {@inheritDoc} */
- @Override public void resumeLogging(WALPointer lastPtr) throws IgniteCheckedException {
- try {
- assert currentHnd == null;
- assert lastPtr == null || lastPtr instanceof FileWALPointer;
-
- FileWALPointer filePtr = (FileWALPointer)lastPtr;
-
- currentHnd = restoreWriteHandle(filePtr);
-
- if (currentHnd.serializer.version() != serializer.version()) {
- if (log.isInfoEnabled())
- log.info("Record serializer version change detected, will start logging with a new WAL record " +
- "serializer to a new WAL segment [curFile=" + currentHnd + ", newVer=" + serializer.version() +
- ", oldVer=" + currentHnd.serializer.version() + ']');
-
- rollOver(currentHnd);
- }
-
- if (mode == WALMode.BACKGROUND) {
- flusher = new QueueFlusher(cctx.igniteInstanceName());
-
- flusher.start();
- }
- }
- catch (StorageException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @SuppressWarnings("TooBroadScope")
- @Override public WALPointer log(WALRecord record) throws IgniteCheckedException, StorageException {
- if (serializer == null || mode == WALMode.NONE)
- return null;
-
- FileWriteHandle current = currentHandle();
-
- // Logging was not resumed yet.
- if (current == null)
- return null;
-
- // Need to calculate record size first.
- record.size(serializer.size(record));
-
- for (; ; current = rollOver(current)) {
- WALPointer ptr = current.addRecord(record);
-
- if (ptr != null) {
- metrics.onWalRecordLogged();
-
- lastWALPtr.set(ptr);
-
- return ptr;
- }
-
- if (isStopping())
- throw new IgniteCheckedException("Stopping.");
- }
- }
-
- /** {@inheritDoc} */
- @Override public void fsync(WALPointer ptr) throws IgniteCheckedException, StorageException {
- if (serializer == null || mode == WALMode.NONE)
- return;
-
- FileWriteHandle cur = currentHandle();
-
- // WAL manager was not started (client node).
- if (cur == null)
- return;
-
- FileWALPointer filePtr = (FileWALPointer)(ptr == null ? lastWALPtr.get() : ptr);
-
- boolean forceFlush = filePtr != null && filePtr.forceFlush();
-
- if (mode == WALMode.BACKGROUND && !forceFlush)
- return;
-
- if (mode == WALMode.LOG_ONLY || forceFlush) {
- cur.flushOrWait(filePtr);
-
- return;
- }
-
- // No need to sync if was rolled over.
- if (filePtr != null && !cur.needFsync(filePtr))
- return;
-
- cur.fsync(filePtr);
- }
-
- /** {@inheritDoc} */
- @Override public WALIterator replay(WALPointer start)
- throws IgniteCheckedException, StorageException {
- assert start == null || start instanceof FileWALPointer : "Invalid start pointer: " + start;
-
- FileWriteHandle hnd = currentHandle();
-
- FileWALPointer end = null;
-
- if (hnd != null)
- end = hnd.position();
-
- return new RecordsIterator(
- cctx,
- walWorkDir,
- walArchiveDir,
- (FileWALPointer)start,
- end,
- psCfg,
- serializer,
- archiver,
- log,
- tlbSize
- );
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
- assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
-
- if (mode == WALMode.NONE)
- return false;
-
- FileArchiver archiver0 = archiver;
-
- if (archiver0 == null)
- throw new IgniteCheckedException("Could not reserve WAL segment: archiver == null");
-
- archiver0.reserve(((FileWALPointer)start).index());
-
- if (!hasIndex(((FileWALPointer)start).index())) {
- archiver0.release(((FileWALPointer)start).index());
-
- return false;
- }
-
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void release(WALPointer start) throws IgniteCheckedException {
- assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
-
- if (mode == WALMode.NONE)
- return;
-
- FileArchiver archiver0 = archiver;
-
- if (archiver0 == null)
- throw new IgniteCheckedException("Could not release WAL segment: archiver == null");
-
- archiver0.release(((FileWALPointer)start).index());
- }
-
- /**
- * @param absIdx Absolulte index to check.
- * @return {@code true} if has this index.
- */
- private boolean hasIndex(long absIdx) {
- String name = FileDescriptor.fileName(absIdx);
-
- boolean inArchive = new File(walArchiveDir, name).exists();
-
- if (inArchive)
- return true;
-
- if (absIdx <= lastArchivedIndex())
- return false;
-
- FileWriteHandle cur = currentHnd;
-
- return cur != null && cur.idx >= absIdx;
- }
-
- /** {@inheritDoc} */
- @Override public int truncate(WALPointer ptr) {
- if (ptr == null)
- return 0;
-
- assert ptr instanceof FileWALPointer : ptr;
-
- FileWALPointer fPtr = (FileWALPointer)ptr;
-
- FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER));
-
- int deleted = 0;
-
- FileArchiver archiver0 = archiver;
-
- for (FileDescriptor desc : descs) {
- // Do not delete reserved segment and any segment after it.
- if (archiver0 != null && archiver0.reserved(desc.idx))
- return deleted;
-
- // We need to leave at least one archived segment to correctly determine the archive index.
- if (desc.idx + 1 < fPtr.index()) {
- if (!desc.file.delete())
- U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
- desc.file.getAbsolutePath());
- else
- deleted++;
-
- // Bump up the oldest archive segment index.
- if (oldestArchiveSegmentIdx < desc.idx)
- oldestArchiveSegmentIdx = desc.idx;
- }
- }
-
- return deleted;
- }
-
- /** {@inheritDoc} */
- @Override public int walArchiveSegments() {
- long oldest = oldestArchiveSegmentIdx;
-
- long lastArchived = archiver.lastArchivedAbsoluteIndex();
-
- if (lastArchived == -1)
- return 0;
-
- int res = (int)(lastArchived - oldest);
-
- return res >= 0 ? res : 0;
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserved(WALPointer ptr) {
- FileWALPointer fPtr = (FileWALPointer)ptr;
-
- FileArchiver archiver0 = archiver;
-
- return archiver0 != null && archiver0.reserved(fPtr.index());
- }
-
- /**
- * Lists files in archive directory and returns the index of last archived file.
- *
- * @return The absolute index of last archived file.
- */
- private long lastArchivedIndex() {
- long lastIdx = -1;
-
- for (File file : walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER)) {
- try {
- long idx = Long.parseLong(file.getName().substring(0, 16));
-
- lastIdx = Math.max(lastIdx, idx);
- }
- catch (NumberFormatException | IndexOutOfBoundsException ignore) {
-
- }
- }
-
- return lastIdx;
- }
-
- /**
- * Lists files in archive directory and returns the index of last archived file.
- *
- * @return The absolute index of last archived file.
- */
- private IgniteBiTuple<Long, Long> scanMinMaxArchiveIndices() {
- long minIdx = Integer.MAX_VALUE;
- long maxIdx = -1;
-
- for (File file : walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER)) {
- try {
- long idx = Long.parseLong(file.getName().substring(0, 16));
-
- minIdx = Math.min(minIdx, idx);
- maxIdx = Math.max(maxIdx, idx);
- }
- catch (NumberFormatException | IndexOutOfBoundsException ignore) {
-
- }
- }
-
- return maxIdx == -1 ? null : F.t(minIdx, maxIdx);
- }
-
- /**
- * Creates a directory specified by the given arguments.
- *
- * @param cfg Configured directory path, may be {@code null}.
- * @param defDir Default directory path, will be used if cfg is {@code null}.
- * @param consId Local node consistent ID.
- * @param msg File description to print out on successful initialization.
- * @return Initialized directory.
- * @throws IgniteCheckedException If failed to initialize directory.
- */
- private File initDirectory(String cfg, String defDir, String consId, String msg) throws IgniteCheckedException {
- File dir;
-
- if (cfg != null) {
- File workDir0 = new File(cfg);
-
- dir = workDir0.isAbsolute() ?
- new File(workDir0, consId) :
- new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), cfg, false), consId);
- }
- else
- dir = new File(U.resolveWorkDirectory(igCfg.getWorkDirectory(), defDir, false), consId);
-
- U.ensureDirectory(dir, msg, log);
-
- return dir;
- }
-
- /**
- * @return Current log segment handle.
- */
- private FileWriteHandle currentHandle() {
- return currentHnd;
- }
-
- /**
- * @param cur Handle that failed to fit the given entry.
- * @return Handle that will fit the entry.
- */
- private FileWriteHandle rollOver(FileWriteHandle cur) throws StorageException, IgniteCheckedException {
- FileWriteHandle hnd = currentHandle();
-
- if (hnd != cur)
- return hnd;
-
- if (hnd.close(true)) {
- FileWriteHandle next = initNextWriteHandle(cur.idx);
-
- boolean swapped = currentHndUpd.compareAndSet(this, hnd, next);
-
- assert swapped : "Concurrent updates on rollover are not allowed";
-
- // Let other threads to proceed with new segment.
- hnd.signalNextAvailable();
- }
- else
- hnd.awaitNext();
-
- return currentHandle();
- }
-
- /**
- * @param lastReadPtr Last read WAL file pointer.
- * @return Initialized file write handle.
- * @throws IgniteCheckedException If failed to initialize WAL write handle.
- */
- private FileWriteHandle restoreWriteHandle(FileWALPointer lastReadPtr) throws IgniteCheckedException {
- long absIdx = lastReadPtr == null ? 0 : lastReadPtr.index();
-
- long segNo = absIdx % psCfg.getWalSegments();
-
- File curFile = new File(walWorkDir, FileDescriptor.fileName(segNo));
-
- int offset = lastReadPtr == null ? 0 : lastReadPtr.fileOffset();
- int len = lastReadPtr == null ? 0 : lastReadPtr.length();
-
- try {
- RandomAccessFile file = new RandomAccessFile(curFile, "rw");
-
- try {
- // readSerializerVersion will change the channel position.
- // This is fine because the FileWriteHandle consitructor will move it
- // to offset + len anyways.
- int serVer = readSerializerVersion(file, curFile, absIdx);
-
- RecordSerializer ser = forVersion(cctx, serVer);
-
- if (log.isInfoEnabled())
- log.info("Resuming logging to WAL segment [file=" + curFile.getAbsolutePath() +
- ", offset=" + offset + ", ver=" + serVer + ']');
-
- FileWriteHandle hnd = new FileWriteHandle(
- file,
- absIdx,
- cctx.igniteInstanceName(),
- offset + len,
- maxWalSegmentSize,
- ser);
-
- if (lastReadPtr == null) {
- HeaderRecord header = new HeaderRecord(serializer.version());
-
- header.size(serializer.size(header));
-
- hnd.addRecord(header);
- }
-
- archiver.currentWalIndex(absIdx);
-
- return hnd;
- }
- catch (IgniteCheckedException | IOException e) {
- file.close();
-
- throw e;
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to restore WAL write handle: " + curFile.getAbsolutePath(), e);
- }
- }
-
- /**
- * Fills the file header for a new segment.
- * Calling this method signals we are done with the segment and it can be archived.
- * If we don't have prepared file yet and achiever is busy this method blocks
- *
- * @param curIdx current segment released by WAL writer
- * @return Initialized file handle.
- * @throws StorageException If IO exception occurred.
- * @throws IgniteCheckedException If failed.
- */
- private FileWriteHandle initNextWriteHandle(long curIdx) throws StorageException, IgniteCheckedException {
- try {
- File nextFile = pollNextFile(curIdx);
-
- if (log.isDebugEnabled())
- log.debug("Switching to a new WAL segment: " + nextFile.getAbsolutePath());
-
- RandomAccessFile file = new RandomAccessFile(nextFile, "rw");
-
- FileWriteHandle hnd = new FileWriteHandle(
- file,
- curIdx + 1,
- cctx.igniteInstanceName(),
- 0,
- maxWalSegmentSize,
- serializer);
-
- HeaderRecord header = new HeaderRecord(serializer.version());
-
- header.size(serializer.size(header));
-
- hnd.addRecord(header);
-
- return hnd;
- }
- catch (IOException e) {
- throw new StorageException(e);
- }
- }
-
- /**
- * Deletes temp files, creates and prepares new; Creates first segment if necessary
- */
- private void checkOrPrepareFiles() throws IgniteCheckedException {
- // Clean temp files.
- {
- File[] tmpFiles = walWorkDir.listFiles(WAL_SEGMENT_TEMP_FILE_FILTER);
-
- if (!F.isEmpty(tmpFiles)) {
- for (File tmp : tmpFiles) {
- boolean deleted = tmp.delete();
-
- if (!deleted)
- throw new IgniteCheckedException("Failed to delete previously created temp file " +
- "(make sure Ignite process has enough rights): " + tmp.getAbsolutePath());
- }
- }
- }
-
- File[] allFiles = walWorkDir.listFiles(WAL_SEGMENT_FILE_FILTER);
-
- if (allFiles.length != 0 && allFiles.length > psCfg.getWalSegments())
- throw new IgniteCheckedException("Failed to initialize wal (work directory contains " +
- "incorrect number of segments) [cur=" + allFiles.length + ", expected=" + psCfg.getWalSegments() + ']');
-
- // Allocate the first segment synchronously. All other segments will be allocated by archiver in background.
- if (allFiles.length == 0) {
- File first = new File(walWorkDir, FileDescriptor.fileName(0));
-
- createFile(first);
- }
- else
- checkFiles(0, false, null);
- }
-
- /**
- * Clears the file with zeros.
- *
- * @param file File to format.
- */
- private void formatFile(File file) throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
-
- try (RandomAccessFile rnd = new RandomAccessFile(file, "rw")) {
- int left = psCfg.getWalSegmentSize();
-
- if (mode == WALMode.DEFAULT) {
- while (left > 0) {
- int toWrite = Math.min(FILL_BUF.length, left);
-
- rnd.write(FILL_BUF, 0, toWrite);
-
- left -= toWrite;
- }
-
- rnd.getChannel().force(false);
- }
- else
- rnd.setLength(0);
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to format WAL segment file: " + file.getAbsolutePath(), e);
- }
- }
-
- /**
- * Creates a file atomically with temp file.
- *
- * @param file File to create.
- * @throws IgniteCheckedException If failed.
- */
- private void createFile(File file) throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Creating new file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
-
- File tmp = new File(file.getParent(), file.getName() + ".tmp");
-
- formatFile(tmp);
-
- try {
- Files.move(tmp.toPath(), file.toPath());
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to move temp file to a regular WAL segment file: " +
- file.getAbsolutePath(), e);
- }
-
- if (log.isDebugEnabled())
- log.debug("Created WAL segment [file=" + file.getAbsolutePath() + ", size=" + file.length() + ']');
- }
-
- /**
- * Retrieves next available file to write WAL data, waiting
- * if necessary for a segment to become available.
- *
- * @param curIdx Current absolute WAL segment index.
- * @return File ready for use as new WAL segment.
- * @throws IgniteCheckedException If failed.
- */
- private File pollNextFile(long curIdx) throws IgniteCheckedException {
- // Signal to archiver that we are done with the segment and it can be archived.
- long absNextIdx = archiver.nextAbsoluteSegmentIndex(curIdx);
-
- long segmentIdx = absNextIdx % psCfg.getWalSegments();
-
- return new File(walWorkDir, FileDescriptor.fileName(segmentIdx));
- }
-
- /**
- * @param ver Serializer version.
- * @return Entry serializer.
- */
- private static RecordSerializer forVersion(GridCacheSharedContext cctx, int ver) throws IgniteCheckedException {
- if (ver <= 0)
- throw new IgniteCheckedException("Failed to create a serializer (corrupted WAL file).");
-
- switch (ver) {
- case 1:
- return new RecordV1Serializer(cctx);
-
- default:
- throw new IgniteCheckedException("Failed to create a serializer with the given version " +
- "(forward compatibility is not supported): " + ver);
- }
- }
-
- /**
- * @return Sorted WAL files descriptors.
- */
- private static FileDescriptor[] scan(File[] allFiles) {
- if (allFiles == null)
- return EMPTY_DESCRIPTORS;
-
- FileDescriptor[] descs = new FileDescriptor[allFiles.length];
-
- for (int i = 0; i < allFiles.length; i++) {
- File f = allFiles[i];
-
- descs[i] = new FileDescriptor(f);
- }
-
- Arrays.sort(descs);
-
- return descs;
- }
-
- /**
- * File archiver operates on absolute segment indexes. For any given absolute segment index N we can calculate
- * the work WAL segment: S(N) = N % psCfg.walSegments.
- * When a work segment is finished, it is given to the archiver. If the absolute index of last archived segment
- * is denoted by A and the absolute index of next segment we want to write is denoted by W, then we can allow
- * write to S(W) if W - A <= walSegments. <br>
- *
- * Monitor of current object is used for notify on:
- * <ul>
- * <li>exception occurred ({@link FileArchiver#cleanException}!=null)</li>
- * <li>stopping thread ({@link FileArchiver#stopped}==true)</li>
- * <li>current file index changed ({@link FileArchiver#curAbsWalIdx})</li>
- * <li>last archived file index was changed ({@link FileArchiver#lastAbsArchivedIdx})</li>
- * <li>some WAL index was removed from {@link FileArchiver#locked} map</li>
- * </ul>
- */
- private class FileArchiver extends Thread {
- /** Exception which occurred during initial creation of files or during archiving WAL segment */
- private IgniteCheckedException cleanException;
-
- /**
- * Absolute current segment index WAL Manger writes to. Guarded by <code>this</code>.
- * Incremented during rollover. Also may be directly set if WAL is resuming logging after start.
- */
- private long curAbsWalIdx = -1;
-
- /** Last archived file index (absolute, 0-based). Guarded by <code>this</code>. */
- private long lastAbsArchivedIdx = -1;
-
- /** current thread stopping advice */
- private volatile boolean stopped;
-
- /** */
- private NavigableMap<Long, Integer> reserved = new TreeMap<>();
-
- /**
- * Maps absolute segment index to locks counter. Lock on segment protects from archiving segment and may
- * come from {@link RecordsIterator} during WAL replay. Map itself is guarded by <code>this</code>.
- */
- private Map<Long, Integer> locked = new HashMap<>();
-
- /**
- *
- */
- private FileArchiver(long lastAbsArchivedIdx) {
- super("wal-file-archiver%" + cctx.igniteInstanceName());
-
- this.lastAbsArchivedIdx = lastAbsArchivedIdx;
- }
-
- /**
- * @return Last archived segment absolute index.
- */
- private synchronized long lastArchivedAbsoluteIndex() {
- return lastAbsArchivedIdx;
- }
-
- /**
- * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
- */
- private void shutdown() throws IgniteInterruptedCheckedException {
- synchronized (this) {
- stopped = true;
-
- notifyAll();
- }
-
- U.join(this);
- }
-
- /**
- * @param curAbsWalIdx Current absolute WAL segment index.
- */
- private void currentWalIndex(long curAbsWalIdx) {
- synchronized (this) {
- this.curAbsWalIdx = curAbsWalIdx;
-
- notifyAll();
- }
- }
-
- /**
- * @param absIdx Index for reservation.
- */
- private synchronized void reserve(long absIdx) {
- Integer cur = reserved.get(absIdx);
-
- if (cur == null)
- reserved.put(absIdx, 1);
- else
- reserved.put(absIdx, cur + 1);
- }
-
- /**
- * @param absIdx Index for reservation.
- * @return {@code True} if index is reserved.
- */
- private synchronized boolean reserved(long absIdx) {
- return locked.containsKey(absIdx) || reserved.floorKey(absIdx) != null;
- }
-
- /**
- * @param absIdx Reserved index.
- */
- private synchronized void release(long absIdx) {
- Integer cur = reserved.get(absIdx);
-
- assert cur != null && cur >= 1 : cur;
-
- if (cur == 1)
- reserved.remove(absIdx);
- else
- reserved.put(absIdx, cur - 1);
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- try {
- allocateRemainingFiles();
- }
- catch (IgniteCheckedException e) {
- synchronized (this) {
- // Stop the thread and report to starter.
- cleanException = e;
-
- notifyAll();
-
- return;
- }
- }
-
- try {
- synchronized (this) {
- while (curAbsWalIdx == -1 && !stopped)
- wait();
-
- if (curAbsWalIdx != 0 && lastAbsArchivedIdx == -1)
- lastAbsArchivedIdx = curAbsWalIdx - 1;
- }
-
- while (!Thread.currentThread().isInterrupted() && !stopped) {
- long toArchive;
-
- synchronized (this) {
- assert lastAbsArchivedIdx <= curAbsWalIdx : "lastArchived=" + lastAbsArchivedIdx +
- ", current=" + curAbsWalIdx;
-
- while (lastAbsArchivedIdx >= curAbsWalIdx - 1 && !stopped)
- wait();
-
- toArchive = lastAbsArchivedIdx + 1;
- }
-
- if (stopped)
- break;
-
- try {
- File workFile = archiveSegment(toArchive);
-
- synchronized (this) {
- while (locked.containsKey(toArchive) && !stopped)
- wait();
-
- // Firstly, format working file
- if (!stopped)
- formatFile(workFile);
-
- // Then increase counter to allow rollover on clean working file
- lastAbsArchivedIdx = toArchive;
-
- notifyAll();
- }
- }
- catch (IgniteCheckedException e) {
- synchronized (this) {
- cleanException = e;
-
- notifyAll();
- }
- }
- }
- }
- catch (InterruptedException ignore) {
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Gets the absolute index of the next WAL segment available to write.
- * Blocks till there are available file to write
- *
- * @param curIdx Current absolute index that we want to increment.
- * @return Next index (curIdx+1) when it is ready to be written.
- * @throws IgniteCheckedException If failed (if interrupted or if exception occurred in the archiver thread).
- */
- private long nextAbsoluteSegmentIndex(long curIdx) throws IgniteCheckedException {
- try {
- synchronized (this) {
- if (cleanException != null)
- throw cleanException;
-
- assert curIdx == curAbsWalIdx;
-
- curAbsWalIdx++;
-
- // Notify archiver thread.
- notifyAll();
-
- while (curAbsWalIdx - lastAbsArchivedIdx > psCfg.getWalSegments() && cleanException == null)
- wait();
-
- return curAbsWalIdx;
- }
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IgniteInterruptedCheckedException(e);
- }
- }
-
- /**
- * @param absIdx Segment absolute index.
- * @return {@code True} if can read, {@code false} if work segment
- */
- @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
- private boolean checkCanReadArchiveOrReserveWorkSegment(long absIdx) {
- synchronized (this) {
- if (lastAbsArchivedIdx >= absIdx)
- return true;
-
- Integer cur = locked.get(absIdx);
-
- cur = cur == null ? 1 : cur + 1;
-
- locked.put(absIdx, cur);
-
- if (log.isDebugEnabled())
- log.debug("Reserved work segment [absIdx=" + absIdx + ", pins=" + cur + ']');
-
- return false;
- }
- }
-
- /**
- * @param absIdx Segment absolute index.
- */
- @SuppressWarnings("NonPrivateFieldAccessedInSynchronizedContext")
- private void releaseWorkSegment(long absIdx) {
- synchronized (this) {
- Integer cur = locked.get(absIdx);
-
- assert cur != null && cur > 0;
-
- if (cur == 1) {
- locked.remove(absIdx);
-
- if (log.isDebugEnabled())
- log.debug("Fully released work segment (ready to archive) [absIdx=" + absIdx + ']');
- }
- else {
- locked.put(absIdx, cur - 1);
-
- if (log.isDebugEnabled())
- log.debug("Partially released work segment [absIdx=" + absIdx + ", pins=" + (cur - 1) + ']');
- }
-
- notifyAll();
- }
- }
-
- /**
- * @param absIdx Absolute index to archive.
- */
- private File archiveSegment(long absIdx) throws IgniteCheckedException {
- long segIdx = absIdx % psCfg.getWalSegments();
-
- File origFile = new File(walWorkDir, FileDescriptor.fileName(segIdx));
-
- String name = FileDescriptor.fileName(absIdx);
-
- File dstTmpFile = new File(walArchiveDir, name + ".tmp");
-
- File dstFile = new File(walArchiveDir, name);
-
- if (log.isDebugEnabled())
- log.debug("Starting to copy WAL segment [absIdx=" + absIdx + ", segIdx=" + segIdx +
- ", origFile=" + origFile.getAbsolutePath() + ", dstFile=" + dstFile.getAbsolutePath() + ']');
-
- try {
- Files.deleteIfExists(dstTmpFile.toPath());
-
- Files.copy(origFile.toPath(), dstTmpFile.toPath());
-
- Files.move(dstTmpFile.toPath(), dstFile.toPath());
-
- if (mode == WALMode.DEFAULT) {
- try (RandomAccessFile f0 = new RandomAccessFile(dstFile, "rw")) {
- f0.getChannel().force(false);
- }
- }
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to archive WAL segment [" +
- "srcFile=" + origFile.getAbsolutePath() +
- ", dstFile=" + dstTmpFile.getAbsolutePath() + ']', e);
- }
-
- if (log.isDebugEnabled())
- log.debug("Copied file [src=" + origFile.getAbsolutePath() +
- ", dst=" + dstFile.getAbsolutePath() + ']');
-
- return origFile;
- }
-
- /**
- *
- */
- private boolean checkStop() {
- return stopped;
- }
-
- /**
- * Background creation of all segments except first. First segment was created in main thread by
- * {@link FileWriteAheadLogManager#checkOrPrepareFiles()}
- */
- private void allocateRemainingFiles() throws IgniteCheckedException {
- checkFiles(1, true, new IgnitePredicate<Integer>() {
- @Override public boolean apply(Integer integer) {
- return !checkStop();
- }
- });
- }
- }
-
- /**
- * Validate files depending on {@link PersistentStoreConfiguration#getWalSegments()} and create if need.
- * Check end when exit condition return false or all files are passed.
- *
- * @param startWith Start with.
- * @param create Flag create file.
- * @param p Predicate Exit condition.
- * @throws IgniteCheckedException if validation or create file fail.
- */
- private void checkFiles(int startWith, boolean create, IgnitePredicate<Integer> p) throws IgniteCheckedException {
- for (int i = startWith; i < psCfg.getWalSegments() && (p == null || (p != null && p.apply(i))); i++) {
- File checkFile = new File(walWorkDir, FileDescriptor.fileName(i));
-
- if (checkFile.exists()) {
- if (checkFile.isDirectory())
- throw new IgniteCheckedException("Failed to initialize WAL log segment (a directory with " +
- "the same name already exists): " + checkFile.getAbsolutePath());
- else if (checkFile.length() != psCfg.getWalSegmentSize() && mode == WALMode.DEFAULT)
- throw new IgniteCheckedException("Failed to initialize WAL log segment " +
- "(WAL segment size change is not supported):" + checkFile.getAbsolutePath());
- }
- else if (create)
- createFile(checkFile);
- }
- }
-
- /**
- * @param rf Random access file.
- * @param file File object.
- * @param idx File index to read.
- * @return Serializer version stored in the file.
- * @throws IOException If failed to read serializer version.
- * @throws IgniteCheckedException If failed to read serializer version.
- */
- private int readSerializerVersion(RandomAccessFile rf, File file, long idx)
- throws IOException, IgniteCheckedException {
- try {
- ByteBuffer buf = ByteBuffer.allocate(RecordV1Serializer.HEADER_RECORD_SIZE);
- buf.order(ByteOrder.nativeOrder());
-
- FileInput in = new FileInput(rf.getChannel(), buf);
-
- // Header record must be agnostic to the serializer version.
- WALRecord rec = serializer.readRecord(in, new FileWALPointer(idx, 0, 0));
-
- if (rec.type() != WALRecord.RecordType.HEADER_RECORD)
- throw new IOException("Missing file header record: " + file.getAbsoluteFile());
-
- return ((HeaderRecord)rec).version();
- }
- catch (SegmentEofException | EOFException ignore) {
- return serializer.version();
- }
- }
-
- /**
- * WAL file descriptor.
- */
- private static class FileDescriptor implements Comparable<FileDescriptor> {
- /** */
- protected final File file;
-
- /** Absolute WAL segment file index */
- protected final long idx;
-
- /**
- * @param file File.
- */
- private FileDescriptor(File file) {
- this(file, null);
- }
-
- /**
- * @param file File.
- * @param idx Absolute WAL segment file index.
- */
- private FileDescriptor(File file, Long idx) {
- this.file = file;
-
- String fileName = file.getName();
-
- assert fileName.endsWith(WAL_SEGMENT_FILE_EXT);
-
- int end = fileName.length() - WAL_SEGMENT_FILE_EXT.length();
-
- this.idx = idx == null ? Long.parseLong(fileName.substring(0, end)) : idx;
- }
-
- /**
- * @param segment Segment index.
- * @return Segment file name.
- */
- private static String fileName(long segment) {
- SB b = new SB();
-
- String segmentStr = Long.toString(segment);
-
- for (int i = segmentStr.length(); i < 16; i++)
- b.a('0');
-
- b.a(segmentStr).a(WAL_SEGMENT_FILE_EXT);
-
- return b.toString();
- }
-
- /**
- * @param segment Segment number as integer.
- * @return Segment number as aligned string.
- */
- private static String segmentNumber(long segment) {
- SB b = new SB();
-
- String segmentStr = Long.toString(segment);
-
- for (int i = segmentStr.length(); i < 16; i++)
- b.a('0');
-
- b.a(segmentStr);
-
- return b.toString();
- }
-
- /** {@inheritDoc} */
- @Override public int compareTo(FileDescriptor o) {
- return Long.compare(idx, o.idx);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (!(o instanceof FileDescriptor))
- return false;
-
- FileDescriptor that = (FileDescriptor)o;
-
- return idx == that.idx;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return (int)(idx ^ (idx >>> 32));
- }
- }
-
- /**
- *
- */
- private abstract static class FileHandle {
- /** */
- protected RandomAccessFile file;
-
- /** */
- protected FileChannel ch;
-
- /** */
- protected final long idx;
-
- /** */
- protected String gridName;
-
- /**
- * @param file File.
- * @param idx Index.
- */
- private FileHandle(RandomAccessFile file, long idx, String gridName) {
- this.file = file;
- this.idx = idx;
- this.gridName = gridName;
-
- ch = file.getChannel();
-
- assert ch != null;
- }
- }
-
- /**
- *
- */
- private static class ReadFileHandle extends FileHandle {
- /** Entry serializer. */
- private RecordSerializer ser;
-
- /** */
- private FileInput in;
-
- /** */
- private boolean workDir;
-
- /**
- * @param file File to read.
- * @param idx File index.
- * @param ser Entry serializer.
- * @param in File input.
- */
- private ReadFileHandle(
- RandomAccessFile file,
- long idx,
- String gridName,
- RecordSerializer ser,
- FileInput in
- ) {
- super(file, idx, gridName);
-
- this.ser = ser;
- this.in = in;
- }
-
- /**
- * @throws IgniteCheckedException If failed to close the WAL segment file.
- */
- public void close() throws IgniteCheckedException {
- try {
- file.close();
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
- }
-
- /**
- * File handle for one log segment.
- */
- @SuppressWarnings("SignalWithoutCorrespondingAwait")
- private class FileWriteHandle extends FileHandle {
- /** */
- private final RecordSerializer serializer;
-
- /** See {@link FileWriteAheadLogManager#maxWalSegmentSize} */
- private final long maxSegmentSize;
-
- /**
- * Accumulated WAL records chain.
- * This reference points to latest WAL record.
- * When writing records chain is iterated from latest to oldest (see {@link WALRecord#previous()})
- * Records from chain are saved into buffer in reverse order
- */
- private final AtomicReference<WALRecord> head = new AtomicReference<>();
-
- /** Position in current file after the end of last written record (incremented after file channel write operation) */
- private volatile long written;
-
- /** */
- private volatile long lastFsyncPos;
-
- /** Environment failure. */
- private volatile Throwable envFailed;
-
- /** Stop guard to provide warranty that only one thread will be successful in calling {@link #close(boolean)}*/
- private final AtomicBoolean stop = new AtomicBoolean(false);
-
- /** */
- private final Lock lock = new ReentrantLock();
-
- /** Condition activated each time writeBuffer() completes. Used to wait previously flushed write to complete */
- private final Condition writeComplete = lock.newCondition();
-
- /** Condition for timed wait of several threads, see {@link PersistentStoreConfiguration#getWalFsyncDelay()} */
- private final Condition fsync = lock.newCondition();
-
- /**
- * Next segment available condition.
- * Protection from "spurious wakeup" is provided by predicate {@link #ch}=<code>null</code>
- */
- private final Condition nextSegment = lock.newCondition();
-
- /**
- * @param file Mapped file to use.
- * @param idx Absolute WAL segment file index for easy access.
- * @param pos Position.
- * @param maxSegmentSize Max segment size.
- * @param serializer Serializer.
- * @throws IOException If failed.
- */
- private FileWriteHandle(
- RandomAccessFile file,
- long idx,
- String gridName,
- long pos,
- long maxSegmentSize,
- RecordSerializer serializer
- ) throws IOException {
- super(file, idx, gridName);
-
- assert serializer != null;
-
- ch.position(pos);
-
- this.maxSegmentSize = maxSegmentSize;
- this.serializer = serializer;
-
- head.set(new FakeRecord(new FileWALPointer(idx, (int)pos, 0)));
- written = pos;
- lastFsyncPos = pos;
- }
-
- /**
- * @param rec Record to be added to record chain as new {@link #head}
- * @return Pointer or null if roll over to next segment is required or already started by other thread.
- * @throws StorageException If failed.
- * @throws IgniteCheckedException If failed.
- */
- @Nullable private WALPointer addRecord(WALRecord rec) throws StorageException, IgniteCheckedException {
- assert rec.size() > 0 || rec.getClass() == FakeRecord.class;
-
- boolean flushed = false;
-
- for (; ; ) {
- WALRecord h = head.get();
-
- long nextPos = nextPosition(h);
-
- // It is important that we read `stop` after `head` in this loop for correct close,
- // because otherwise we will have a race on the last flush in close.
- if (nextPos + rec.size() >= maxSegmentSize || stop.get()) {
- // Can not write to this segment, need to switch to the next one.
- return null;
- }
-
- int newChainSize = h.chainSize() + rec.size();
-
- if (newChainSize > tlbSize && !flushed) {
- boolean res = h.previous() == null || flush(h);
-
- if (rec.size() > tlbSize)
- flushed = res;
-
- continue;
- }
-
- rec.chainSize(newChainSize);
- rec.previous(h);
-
- FileWALPointer ptr = new FileWALPointer(
- idx,
- (int)nextPos,
- rec.size(),
- // We need to force checkpoint records into file in BACKGROUND WALMode.
- mode == WALMode.BACKGROUND && rec instanceof CheckpointRecord);
-
- rec.position(ptr);
-
- if (head.compareAndSet(h, rec))
- return ptr;
- }
- }
-
- /**
- * @param rec Record.
- * @return Position for the next record.
- */
- private long nextPosition(WALRecord rec) {
- return recordOffset(rec) + rec.size();
- }
-
- /**
- * Flush or wait for concurrent flush completion.
- *
- * @param ptr Pointer.
- * @throws IgniteCheckedException If failed.
- */
- private void flushOrWait(FileWALPointer ptr) throws IgniteCheckedException {
- long expWritten;
-
- if (ptr != null) {
- // If requested obsolete file index, it must be already flushed by close.
- if (ptr.index() != idx)
- return;
-
- expWritten = ptr.fileOffset();
- }
- else // We read head position before the flush because otherwise we can get wrong position.
- expWritten = recordOffset(head.get());
-
- if (flush(ptr))
- return;
-
- // Spin-wait for a while before acquiring the lock.
- for (int i = 0; i < 64; i++) {
- if (written >= expWritten)
- return;
- }
-
- // If we did not flush ourselves then await for concurrent flush to complete.
- lock.lock();
-
- try {
- while (written < expWritten && envFailed == null)
- U.await(writeComplete);
- }
- finally {
- lock.unlock();
- }
- }
-
- /**
- * @param ptr Pointer.
- * @return {@code true} If the flush really happened.
- * @throws IgniteCheckedException If failed.
- * @throws StorageException If failed.
- */
- private boolean flush(FileWALPointer ptr) throws IgniteCheckedException, StorageException {
- if (ptr == null) { // Unconditional flush.
- for (; ; ) {
- WALRecord expHead = head.get();
-
- if (expHead.previous() == null) {
- assert expHead instanceof FakeRecord;
-
- return false;
- }
-
- if (flush(expHead))
- return true;
- }
- }
-
- assert ptr.index() == idx;
-
- for (; ; ) {
- WALRecord h = head.get();
-
- // If current chain begin position is greater than requested, then someone else flushed our changes.
- if (chainBeginPosition(h) > ptr.fileOffset())
- return false;
-
- if (flush(h))
- return true; // We are lucky.
- }
- }
-
- /**
- * @param h Head of the chain.
- * @return Chain begin position.
- */
- private long chainBeginPosition(WALRecord h) {
- return recordOffset(h) + h.size() - h.chainSize();
- }
-
- /**
- * @param expHead Expected head of chain. If head was changed, flush is not performed in this thread
- * @throws IgniteCheckedException If failed.
- * @throws StorageException If failed.
- */
- private boolean flush(WALRecord expHead) throws StorageException, IgniteCheckedException {
- if (expHead.previous() == null) {
- assert expHead instanceof FakeRecord;
-
- return false;
- }
-
- // Fail-fast before CAS.
- checkEnvironment();
-
- if (!head.compareAndSet(expHead, new FakeRecord(new FileWALPointer(idx, (int)nextPosition(expHead), 0))))
- return false;
-
- // At this point we grabbed the piece of WAL chain.
- // Any failure in this code must invalidate the environment.
- try {
- // We can safely allow other threads to start building next chains while we are doing flush here.
- ByteBuffer buf;
-
- boolean tmpBuf = false;
-
- if (expHead.chainSize() > tlbSize) {
- buf = GridUnsafe.allocateBuffer(expHead.chainSize());
-
- tmpBuf = true; // We need to manually release this temporary direct buffer.
- }
- else
- buf = tlb.get();
-
- try {
- long pos = fillBuffer(buf, expHead);
-
- writeBuffer(pos, buf);
- }
- finally {
- if (tmpBuf)
- GridUnsafe.freeBuffer(buf);
- }
-
- return true;
- }
- catch (Throwable e) {
- invalidateEnvironment(e);
-
- throw e;
- }
- }
-
- /**
- * Serializes WAL records chain to provided byte buffer
- * @param buf Buffer, will be filled with records chain from end to beginning
- * @param head Head of the chain to write to the buffer.
- * @return Position in file for this buffer.
- * @throws IgniteCheckedException If failed.
- */
- private long fillBuffer(ByteBuffer buf, WALRecord head) throws IgniteCheckedException {
- final int limit = head.chainSize();
-
- assert limit <= buf.capacity();
-
- buf.rewind();
- buf.limit(limit);
-
- do {
- buf.position(head.chainSize() - head.size());
- buf.limit(head.chainSize()); // Just to make sure that serializer works in bounds.
-
- try {
- serializer.writeRecord(head, buf);
- }
- catch (RuntimeException e) {
- throw new IllegalStateException("Failed to write record: " + head, e);
- }
-
- assert !buf.hasRemaining() : "Reported record size is greater than actual: " + head;
-
- head = head.previous();
- }
- while (head.previous() != null);
-
- assert head instanceof FakeRecord : head.getClass();
-
- buf.rewind();
- buf.limit(limit);
-
- return recordOffset(head);
- }
-
- /**
- * Non-blocking check if this pointer needs to be sync'ed.
- *
- * @param ptr WAL pointer to check.
- * @return {@code False} if this pointer has been already sync'ed.
- */
- private boolean needFsync(FileWALPointer ptr) {
- // If index has changed, it means that the log was rolled over and already sync'ed.
- // If requested position is smaller than last sync'ed, it also means all is good.
- // If position is equal, then our record is the last not synced.
- return idx == ptr.index() && lastFsyncPos <= ptr.fileOffset();
- }
-
- /**
- * @return Pointer to the end of the last written record (probably not fsync-ed).
- */
- private FileWALPointer position() {
- lock.lock();
-
- try {
- return new FileWALPointer(idx, (int)written, 0);
- }
- finally {
- lock.unlock();
- }
- }
-
- /**
- * @param ptr Pointer to sync.
- * @throws StorageException If failed.
- */
- private void fsync(FileWALPointer ptr) throws StorageException, IgniteCheckedException {
- lock.lock();
-
- try {
- if (ptr != null) {
- if (!needFsync(ptr))
- return;
-
- if (fsyncDelay > 0 && !stop.get()) {
- // Delay fsync to collect as many updates as possible: trade latency for throughput.
- U.await(fsync, fsyncDelay, TimeUnit.NANOSECONDS);
-
- if (!needFsync(ptr))
- return;
- }
- }
-
- flushOrWait(ptr);
-
- if (lastFsyncPos != written) {
- assert lastFsyncPos < written; // Fsync position must be behind.
-
- boolean metricsEnabled = metrics.metricsEnabled();
-
- long start = metricsEnabled ? System.nanoTime() : 0;
-
- try {
- ch.force(false);
- }
- catch (IOException e) {
- throw new StorageException(e);
- }
-
- lastFsyncPos = written;
-
- if (fsyncDelay > 0)
- fsync.signalAll();
-
- long end = metricsEnabled ? System.nanoTime() : 0;
-
- if (metricsEnabled)
- metrics.onFsync(end - start);
- }
- }
- finally {
- lock.unlock();
- }
- }
-
- /**
- * @return {@code true} If this thread actually closed the segment.
- * @throws IgniteCheckedException If failed.
- * @throws StorageException If failed.
- */
- private boolean close(boolean rollOver) throws IgniteCheckedException, StorageException {
- if (stop.compareAndSet(false, true)) {
- // Here we can be sure that no other records will be added and this fsync will be the last.
- if (mode == WALMode.DEFAULT)
- fsync(null);
- else
- flushOrWait(null);
-
- try {
- if (rollOver && written < (maxSegmentSize - 1)) {
- ByteBuffer allocate = ByteBuffer.allocate(1);
- allocate.put((byte) WALRecord.RecordType.SWITCH_SEGMENT_RECORD.ordinal());
-
- ch.write(allocate, written);
-
- if (mode == WALMode.DEFAULT)
- ch.force(false);
- }
-
- ch.close();
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- if (log.isDebugEnabled())
- log.debug("Closed WAL write handle [idx=" + idx + "]");
-
- return true;
- }
-
- return false;
- }
-
- /**
- * Signals next segment available to wake up other worker threads waiting for WAL to write
- */
- private void signalNextAvailable() {
- lock.lock();
-
- try {
- assert head.get() instanceof FakeRecord: "head";
- assert written == lastFsyncPos || mode != WALMode.DEFAULT :
- "fsync [written=" + written + ", lastFsync=" + lastFsyncPos + ']';
-
- ch = null;
-
- nextSegment.signalAll();
- }
- finally {
- lock.unlock();
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void awaitNext() throws IgniteCheckedException {
- lock.lock();
-
- try {
- while (ch != null)
- U.await(nextSegment);
- }
- finally {
- lock.unlock();
- }
- }
-
- /**
- * @param pos Position in file to start write from.
- * May be checked against actual position to wait previous writes to complete
- * @param buf Buffer to write to file
- * @throws StorageException If failed.
- * @throws IgniteCheckedException If failed.
- */
- @SuppressWarnings("TooBroadScope")
- private void writeBuffer(long pos, ByteBuffer buf) throws StorageException, IgniteCheckedException {
- boolean interrupted = false;
-
- lock.lock();
-
- try {
- assert ch != null : "Writing to a closed segment.";
-
- checkEnvironment();
-
- long lastLogged = U.currentTimeMillis();
-
- long logBackoff = 2_000;
-
- // If we were too fast, need to wait previous writes to complete.
- while (written != pos) {
- assert written < pos : "written = " + written + ", pos = " + pos; // No one can write further than we are now.
-
- // Permutation occurred between blocks write operations.
- // Order of acquiring lock is not the same as order of write.
- long now = U.currentTimeMillis();
-
- if (now - lastLogged >= logBackoff) {
- if (logBackoff < 60 * 60_000)
- logBackoff *= 2;
-
- U.warn(log, "Still waiting for a concurrent write to complete [written=" + written +
- ", pos=" + pos + ", lastFsyncPos=" + lastFsyncPos + ", stop=" + stop.get() +
- ", actualPos=" + safePosition() + ']');
-
- lastLogged = now;
- }
-
- try {
- writeComplete.await(2, TimeUnit.SECONDS);
- }
- catch (InterruptedException ignore) {
- interrupted = true;
- }
-
- checkEnvironment();
- }
-
- // Do the write.
- int size = buf.remaining();
-
- assert size > 0 : size;
-
- try {
- assert written == ch.position();
-
- do {
- ch.write(buf);
- }
- while (buf.hasRemaining());
-
- written += size;
-
- metrics.onWalBytesWritten(size);
-
- assert written == ch.position();
- }
- catch (IOException e) {
- invalidateEnvironmentLocked(e);
-
- throw new StorageException(e);
- }
- }
- finally {
- writeComplete.signalAll();
-
- lock.unlock();
-
- if (interrupted)
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * @param e Exception to set as a cause for all further operations.
- */
- private void invalidateEnvironment(Throwable e) {
- lock.lock();
-
- try {
- invalidateEnvironmentLocked(e);
- }
- finally {
- writeComplete.signalAll();
-
- lock.unlock();
- }
- }
-
- /**
- * @param e Exception to set as a cause for all further operations.
- */
- private void invalidateEnvironmentLocked(Throwable e) {
- if (envFailed == null) {
- envFailed = e;
-
- U.error(log, "IO error encountered while running WAL flush. All further operations will be failed and " +
- "local node will be stopped.", e);
-
- new Thread() {
- @Override public void run() {
- G.stop(gridName, true);
- }
- }.start();
- }
- }
-
- /**
- * @throws StorageException If environment is no longer valid and we missed a WAL write.
- */
- private void checkEnvironment() throws StorageException {
- if (envFailed != null)
- throw new StorageException("Failed to flush WAL buffer (environment was invalidated by a " +
- "previous error)", envFailed);
- }
-
- /**
- * @return Safely reads current position of the file channel as String. Will return "null" if channel is null.
- */
- private String safePosition() {
- FileChannel ch = this.ch;
-
- if (ch == null)
- return "null";
-
- try {
- return String.valueOf(ch.position());
- }
- catch (IOException e) {
- return "{Failed to read channel position: " + e.getMessage() + "}";
- }
- }
- }
-
- /**
- * Gets WAL record offset relative to the WAL segment file beginning.
- *
- * @param rec WAL record.
- * @return File offset.
- */
- private static int recordOffset(WALRecord rec) {
- FileWALPointer ptr = (FileWALPointer)rec.position();
-
- assert ptr != null;
-
- return ptr.fileOffset();
- }
-
- /**
- * Fake record is zero-sized record, which is not stored into file.
- * Fake record is used for storing position in file {@link WALRecord#position()}.
- * Fake record is allowed to have no previous record.
- */
- private static final class FakeRecord extends WALRecord {
- /**
- * @param pos Position.
- */
- FakeRecord(FileWALPointer pos) {
- position(pos);
- }
-
- /** {@inheritDoc} */
- @Override public RecordType type() {
- return null;
- }
- }
-
- /**
- * Iterator over WAL-log.
- */
- private static class RecordsIterator extends GridCloseableIteratorAdapter<IgniteBiTuple<WALPointer, WALRecord>>
- implements WALIterator {
- /** */
- private static final long serialVersionUID = 0L;
- /** */
- private final File walWorkDir;
-
- /** */
- private final File walArchiveDir;
-
- /** */
- private final FileArchiver archiver;
-
- /** */
- private final PersistentStoreConfiguration psCfg;
-
- /** */
- private final RecordSerializer serializer;
-
- /** */
- private final GridCacheSharedContext cctx;
-
- /** */
- private FileWALPointer start;
-
- /** */
- private FileWALPointer end;
-
- /** */
- private IgniteBiTuple<WALPointer, WALRecord> curRec;
-
- /** */
- private long curIdx = -1;
-
- /** */
- private ReadFileHandle curHandle;
-
- /** */
- private ByteBuffer buf;
-
- /** */
- private IgniteLogger log;
-
- /**
- * @param cctx Shared context.
- * @param walWorkDir WAL work dir.
- * @param walArchiveDir WAL archive dir.
- * @param start Optional start pointer.
- * @param end Optional end pointer.
- * @param psCfg Database configuration.
- * @param serializer Serializer.
- * @param archiver Archiver.
- * @throws IgniteCheckedException If failed to initialize WAL segment.
- */
- private RecordsIterator(
- GridCacheSharedContext cctx,
- File walWorkDir,
- File walArchiveDir,
- FileWALPointer start,
- FileWALPointer end,
- PersistentStoreConfiguration psCfg,
- RecordSerializer serializer,
- FileArchiver archiver,
- IgniteLogger log,
- int tlbSize
- ) throws IgniteCheckedException {
- this.cctx = cctx;
- this.walWorkDir = walWorkDir;
- this.walArchiveDir = walArchiveDir;
- this.psCfg = psCfg;
- this.serializer = serializer;
- this.archiver = archiver;
- this.start = start;
- this.end = end;
- this.log = log;
-
- int buffSize = Math.min(16 * tlbSize, psCfg.getWalRecordIteratorBufferSize());
-
- // Do not allocate direct buffer for iterator.
- buf = ByteBuffer.allocate(buffSize);
- buf.order(ByteOrder.nativeOrder());
-
- init();
-
- advance();
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteBiTuple<WALPointer, WALRecord> onNext() throws IgniteCheckedException {
- IgniteBiTuple<WALPointer, WALRecord> ret = curRec;
-
- advance();
-
- return ret;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean onHasNext() throws IgniteCheckedException {
- return curRec != null;
- }
-
- /** {@inheritDoc} */
- @Override protected void onClose() throws IgniteCheckedException {
- curRec = null;
-
- if (curHandle != null) {
- curHandle.close();
-
- if (curHandle.workDir)
- releaseWorkSegment(curIdx);
-
- curHandle = null;
- }
-
- curIdx = Integer.MAX_VALUE;
- }
-
- /**
- * @throws IgniteCheckedException If failed to initialize first file handle.
- */
- private void init() throws IgniteCheckedException {
- FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_FILTER));
-
- if (start != null) {
- if (!F.isEmpty(descs)) {
- if (descs[0].idx > start.index())
- throw new IgniteCheckedException("WAL history is too short " +
- "[descs=" + Arrays.asList(descs) + ", start=" + start + ']');
-
- for (FileDescriptor desc : descs) {
- if (desc.idx == start.index()) {
- curIdx = start.index();
-
- break;
- }
- }
-
- if (curIdx == -1) {
- long lastArchived = descs[descs.length - 1].idx;
-
- if (lastArchived > start.index())
- throw new IgniteCheckedException("WAL history is corrupted (segment is missing): " + start);
-
- // This pointer may be in work files because archiver did not
- // copy the file yet, check that it is not too far forward.
- curIdx = start.index();
- }
- }
- else {
- // This means that whole checkpoint history fits in one segment in WAL work directory.
- // Will start from this index right away.
- curIdx = start.index();
- }
- }
- else
- curIdx = !F.isEmpty(descs) ? descs[0].idx : 0;
-
- curIdx--;
-
- if (log.isDebugEnabled())
- log.debug("Initialized WAL cursor [start=" + start + ", end=" + end + ", curIdx=" + curIdx + ']');
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void advance() throws IgniteCheckedException {
- while (true) {
- advanceRecord();
-
- if (curRec != null)
- return;
- else {
- advanceSegment();
-
- if (curHandle == null)
- return;
- }
- }
- }
-
- /**
- *
- */
- private void advanceRecord() {
- try {
- ReadFileHandle hnd = curHandle;
-
- if (hnd != null) {
- RecordSerializer ser = hnd.ser;
-
- int pos = (int)hnd.in.position();
-
- FileWALPointer ptr = new FileWALPointer(hnd.idx, pos, 0);
-
- WALRecord rec = ser.readRecord(hnd.in, ptr);
-
- ptr.length(rec.size());
-
- curRec = new IgniteBiTuple<WALPointer, WALRecord>(ptr, rec);
- }
- }
- catch (IOException | IgniteCheckedException e) {
- if (!(e instanceof SegmentEofException)) {
- if (log.isInfoEnabled())
- log.info("Stopping WAL iteration due to an exception: " + e.getMessage());
- }
-
- curRec = null;
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void advanceSegment() throws IgniteCheckedException {
- ReadFileHandle cur0 = curHandle;
-
- if (cur0 != null) {
- cur0.close();
-
- if (cur0.workDir)
- releaseWorkSegment(cur0.idx);
-
- curHandle = null;
- }
-
- // We are past the end marker.
- if (end != null && curIdx + 1 > end.index())
- return;
-
- curIdx++;
-
- FileDescriptor fd;
-
- boolean readArchive = canReadArchiveOrReserveWork(curIdx);
-
- if (readArchive) {
- fd = new FileDescriptor(new File(walArchiveDir,
- FileDescriptor.fileName(curIdx)));
- }
- else {
- long workIdx = curIdx % psCfg.getWalSegments();
-
- fd = new FileDescriptor(
- new File(walWorkDir, FileDescriptor.fileName(workIdx)),
- curIdx);
- }
-
- if (log.isDebugEnabled())
- log.debug("Reading next file [absIdx=" + curIdx + ", file=" + fd.file.getAbsolutePath() + ']');
-
- assert fd != null;
-
- try {
- curHandle = initReadHandle(fd, start != null && curIdx == start.index() ? start : null);
- }
- catch (FileNotFoundException e) {
- if (readArchive)
- throw new IgniteCheckedException("Missing WAL segment in the archive", e);
- else
- curHandle = null;
- }
-
- if (curHandle != null)
- curHandle.workDir = !readArchive;
- else
- releaseWorkSegment(curIdx);
-
- curRec = null;
- }
-
- /**
- * @param desc File descriptor.
- * @param start Optional start pointer.
- * @return Initialized file handle.
- * @throws FileNotFoundException If segment file is missing.
- * @throws IgniteCheckedException If initialized failed due to another unexpected error.
- */
- private ReadFileHandle initReadHandle(FileDescriptor desc, FileWALPointer start)
- throws IgniteCheckedException, FileNotFoundException {
- try {
- RandomAccessFile rf = new RandomAccessFile(desc.file, "r");
-
- try {
- FileChannel channel = rf.getChannel();
- FileInput in = new FileInput(channel, buf);
-
- // Header record must be agnostic to the serializer version.
- WALRecord rec = serializer.readRecord(in,
- new FileWALPointer(desc.idx, (int)channel.position(), 0));
-
- if (rec == null)
- return null;
-
- if (rec.type() != WALRecord.RecordType.HEADER_RECORD)
- throw new IOException("Missing file header record: " + desc.file.getAbsoluteFile());
-
- int ver = ((HeaderRecord)rec).version();
-
- RecordSerializer ser = forVersion(cctx, ver);
-
- if (start != null && desc.idx == start.index())
- in.seek(start.fileOffset());
-
- return new ReadFileHandle(rf, desc.idx, cctx.igniteInstanceName(), ser, in);
- }
- catch (SegmentEofException | EOFException ignore) {
- try {
- rf.close();
- }
- catch (IOException ce) {
- throw new IgniteCheckedException(ce);
- }
-
- return null;
- }
- catch (IOException | IgniteCheckedException e) {
- try {
- rf.close();
- }
- catch (IOException ce) {
- e.addSuppressed(ce);
- }
-
- throw e;
- }
- }
- catch (FileNotFoundException e) {
- throw e;
- }
- catch (IOException e) {
- throw new IgniteCheckedException(
- "Failed to initialize WAL segment: " + desc.file.getAbsolutePath(), e);
- }
- }
-
- /**
- * @param absIdx Absolute index to check.
- * @return {@code True} if we can safely read the archive, {@code false} if the segment has not been
- * archived yet. In this case the corresponding work segment is reserved (will not be deleted until
- * release).
- */
- private boolean canReadArchiveOrReserveWork(long absIdx) {
- return archiver != null && archiver.checkCanReadArchiveOrReserveWorkSegment(absIdx);
- }
-
- /**
- * @param absIdx Absolute index to release.
- */
- private void releaseWorkSegment(long absIdx) {
- if (archiver != null)
- archiver.releaseWorkSegment(absIdx);
- }
- }
-
- /**
- * Periodically flushes current file handle for {@link WALMode#BACKGROUND} WALMode.
- */
- private class QueueFlusher extends Thread {
- /** */
- private volatile boolean stopped;
-
- /**
- * @param gridName Grid name.
- */
- private QueueFlusher(String gridName) {
- super("wal-queue-flusher-#" + gridName);
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- while (!stopped) {
- long wakeup = U.currentTimeMillis() + flushFreq;
-
- LockSupport.parkUntil(wakeup);
-
- FileWriteHandle hnd = currentHandle();
-
- try {
- hnd.flush(hnd.head.get());
- }
- catch (IgniteCheckedException e) {
- U.warn(log, "Failed to flush WAL record queue", e);
- }
- }
- }
-
- /**
- * Signals stop, wakes up thread and waiting until completion.
- */
- private void shutdown() {
- stopped = true;
-
- LockSupport.unpark(this);
-
- try {
- join();
- }
- catch (InterruptedException ignore) {
- // Got interrupted while waiting for flusher to shutdown.
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/RecordSerializer.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/RecordSerializer.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/RecordSerializer.java
deleted file mode 100644
index c929789..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/wal/RecordSerializer.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.wal;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-
-/**
- * Record serializer.
- */
-public interface RecordSerializer {
- /**
- * @return serializer version
- */
- public int version();
-
- /**
- * @param record Record.
- * @return Size in bytes.
- */
- public int size(WALRecord record) throws IgniteCheckedException;
-
- /**
- * @param record Entry to write.
- * @param buf Buffer.
- */
- public void writeRecord(WALRecord record, ByteBuffer buf) throws IgniteCheckedException;
-
- /**
- * @param in Data input to read data from.
- * @return Read entry.
- */
- public WALRecord readRecord(FileInput in, WALPointer expPtr) throws IOException, IgniteCheckedException;
-}
[04/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/FullPageIdTableTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/FullPageIdTableTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/FullPageIdTableTest.java
deleted file mode 100644
index df212be..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/FullPageIdTableTest.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.internal.mem.DirectMemoryRegion;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.processors.cache.database.pagemem.FullPageIdTable;
-import org.apache.ignite.internal.util.typedef.CI2;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.logger.java.JavaLogger;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class FullPageIdTableTest extends GridCommonAbstractTest {
- /** */
- private static final int CACHE_ID_RANGE = 10;
-
- /** */
- private static final int PAGE_ID_RANGE = 1000;
-
- /**
- * @throws Exception if failed.
- */
- public void testRandomOperations() throws Exception {
- long mem = FullPageIdTable.requiredMemory(CACHE_ID_RANGE * PAGE_ID_RANGE);
-
- UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
-
- prov.initialize(new long[] {mem});
-
- DirectMemoryRegion region = prov.nextRegion();
-
- try {
- long seed = U.currentTimeMillis();
-
- info("Seed: " + seed + "L; //");
-
- Random rnd = new Random(seed);
-
- FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
-
- Map<FullPageId, Long> check = new HashMap<>();
-
- for (int i = 0; i < 10_000; i++) {
- int cacheId = rnd.nextInt(CACHE_ID_RANGE) + 1;
- int pageId = rnd.nextInt(PAGE_ID_RANGE);
-
- FullPageId fullId = new FullPageId(pageId, cacheId);
-
- boolean put = rnd.nextInt(3) != -1;
-
- if (put) {
- long val = rnd.nextLong();
-
- tbl.put(cacheId, pageId, val, 0);
- check.put(fullId, val);
- }
- else {
- tbl.remove(cacheId, pageId, 0);
- check.remove(fullId);
- }
-
- verifyLinear(tbl, check);
-
- if (i > 0 && i % 1000 == 0)
- info("Done: " + i);
- }
- }
- finally {
- prov.shutdown();
- }
- }
-
- /**
- * @param tbl Table to check.
- * @param check Expected mapping.
- */
- private void verifyLinear(FullPageIdTable tbl, Map<FullPageId, Long> check) {
- final Map<FullPageId, Long> collector = new HashMap<>();
-
- tbl.visitAll(new CI2<FullPageId, Long>() {
- @Override public void apply(FullPageId fullId, Long val) {
- if (collector.put(fullId, val) != null)
- throw new AssertionError("Duplicate full page ID mapping: " + fullId);
- }
- });
-
- assertEquals("Size check failed", check.size(), collector.size());
-
- for (Map.Entry<FullPageId, Long> entry : check.entrySet())
- assertEquals("Mapping comparison failed for key: " + entry.getKey(),
- entry.getValue(), collector.get(entry.getKey()));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/MetadataStoragePageMemoryImplTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/MetadataStoragePageMemoryImplTest.java
deleted file mode 100644
index a6e241c..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/MetadataStoragePageMemoryImplTest.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-
-/**
- *
- */
-public class MetadataStoragePageMemoryImplTest extends MetadataStorageSelfTest{
- /** Make sure page is small enough to trigger multiple pages in a linked list. */
- public static final int PAGE_SIZE = 1024;
-
- /** */
- private static File allocationPath;
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
- }
-
- /**
- * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
- * new empty page memory.
- * @return Page memory instance.
- */
- @Override protected PageMemory memory(boolean clean) throws Exception {
- long[] sizes = new long[10];
-
- for (int i = 0; i < sizes.length; i++)
- sizes[i] = 1024 * 1024;
-
- DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
-
- GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
- new GridTestKernalContext(log),
- null,
- null,
- null,
- new NoOpPageStoreManager(),
- new NoOpWALManager(),
- new IgniteCacheDatabaseSharedManager(),
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null
- );
-
- return new PageMemoryImpl(
- provider, sizes,
- sharedCtx,
- PAGE_SIZE,
- new CIX3<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
- assert false : "No evictions should happen during the test";
- }
- },
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
- }
- }, new CheckpointLockStateChecker() {
- @Override public boolean checkpointLockIsHeldByThread() {
- return true;
- }
- },
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpPageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpPageStoreManager.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpPageStoreManager.java
deleted file mode 100644
index 144bbef..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpPageStoreManager.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.nio.ByteBuffer;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.lang.IgniteFuture;
-
-/**
- *
- */
-public class NoOpPageStoreManager implements IgnitePageStoreManager {
- /** */
- private ConcurrentMap<FullPageId, AtomicInteger> allocators = new ConcurrentHashMap<>();
-
- /** {@inheritDoc} */
- @Override public void beginRecover() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void finishRecover() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void initializeForCache(CacheGroupDescriptor grpDesc,
- StoredCacheData cacheData) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void shutdownForCacheGroup(CacheGroupContext grp, boolean destroy) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onPartitionCreated(int grpId, int partId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onPartitionDestroyed(int cacheId, int partId, int tag) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void read(int cacheId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException {
-
- }
-
- /** {@inheritDoc} */
- @Override public boolean exists(int cacheId, int partId) throws IgniteCheckedException {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void readHeader(int cacheId, int partId, ByteBuffer buf) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void write(int cacheId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void sync(int cacheId, int partId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void ensure(int cacheId, int partId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public long pageOffset(int cacheId, long pageId) throws IgniteCheckedException {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
- long root = PageIdUtils.pageId(partId, flags, 0);
-
- FullPageId fullId = new FullPageId(root, cacheId);
-
- AtomicInteger allocator = allocators.get(fullId);
-
- if (allocator == null)
- allocator = F.addIfAbsent(allocators, fullId, new AtomicInteger(1));
-
- return PageIdUtils.pageId(partId, flags, allocator.getAndIncrement());
- }
-
- /** {@inheritDoc} */
- @Override public int pages(int cacheId, int partId) throws IgniteCheckedException {
- long root = PageIdUtils.pageId(partId, (byte)0, 0);
-
- FullPageId fullId = new FullPageId(root, cacheId);
-
- AtomicInteger allocator = allocators.get(fullId);
-
- if (allocator == null)
- allocator = F.addIfAbsent(allocators, fullId, new AtomicInteger(2));
-
- return allocator.get();
- }
-
- /** {@inheritDoc} */
- @Override public long metaPageId(int cacheId) {
- return 1;
- }
-
- /** {@inheritDoc} */
- @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void stop(boolean cancel) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStart(boolean reconnect) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStop(boolean cancel) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onDisconnected(IgniteFuture reconnectFut) {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void printMemoryStats() {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public Map<String, StoredCacheData> readCacheConfigurations() throws IgniteCheckedException {
- return Collections.emptyMap();
- }
-
- /** {@inheritDoc} */
- @Override public void storeCacheData(CacheGroupDescriptor grpDesc,
- StoredCacheData cacheData) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasIndexStore(int grpId) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) {
-
- }
-
- /** {@inheritDoc} */
- @Override public void onDeActivate(GridKernalContext kctx) {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpWALManager.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpWALManager.java
deleted file mode 100644
index f3e1cd3..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/NoOpWALManager.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.lang.IgniteFuture;
-
-/**
- *
- */
-public class NoOpWALManager implements IgniteWriteAheadLogManager {
- /** {@inheritDoc} */
- @Override public boolean isAlwaysWriteFullPages() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public boolean isFullSync() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void resumeLogging(WALPointer ptr) throws IgniteCheckedException {
-
- }
-
- /** {@inheritDoc} */
- @Override public WALPointer log(WALRecord entry) throws IgniteCheckedException, StorageException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void fsync(WALPointer ptr) throws IgniteCheckedException, StorageException {
-
- }
-
- /** {@inheritDoc} */
- @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException {
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void release(WALPointer start) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public int truncate(WALPointer ptr) {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserved(WALPointer ptr) {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
-
- }
-
- /** {@inheritDoc} */
- @Override public void stop(boolean cancel) {
-
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStart(boolean reconnect) throws IgniteCheckedException {
-
- }
-
- /** {@inheritDoc} */
- @Override public void onKernalStop(boolean cancel) {
-
- }
-
- /** {@inheritDoc} */
- @Override public void onDisconnected(IgniteFuture reconnectFut) {
-
- }
-
- /** {@inheritDoc} */
- @Override public void printMemoryStats() {
-
- }
-
- /** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
-
- }
-
- /** {@inheritDoc} */
- @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
-
- }
-
- /** {@inheritDoc} */
- @Override public int walArchiveSegments() {
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageIdDistributionTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageIdDistributionTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageIdDistributionTest.java
deleted file mode 100644
index a37f981..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageIdDistributionTest.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.io.FileOutputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import org.apache.ignite.internal.mem.DirectMemoryRegion;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.processors.cache.database.pagemem.FullPageIdTable;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.logger.java.JavaLogger;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class PageIdDistributionTest extends GridCommonAbstractTest {
- /** */
- private static final int[] CACHE_IDS = new int[] {
- CU.cacheId("partitioned1"),
- CU.cacheId("partitioned2"),
- CU.cacheId("partitioned3"),
- CU.cacheId("partitioned4"),
- CU.cacheId("replicated1"),
- CU.cacheId("replicated2"),
- CU.cacheId("replicated3"),
- CU.cacheId("replicated4"),
- };
-
- /** */
- private static final int PARTS = 1024;
-
- /** */
- private static final int PAGES = 10240;
-
- /**
- *
- */
- public void testDistributions() {
- printPageIdDistribution(
- CU.cacheId("partitioned"), 1024, 30_000, 32, 2.5f);
-
- printPageIdDistribution(
- CU.cacheId("partitioned"), 1024, 30_000, 64, 2.5f);
-
- printPageIdDistribution(
- CU.cacheId(null), 1024, 30_000, 32, 2.5f);
- }
-
- /**
- * @param cacheId Cache id.
- * @param parts Parts.
- * @param pagesPerPartition Pages per partition.
- * @param segments Segments.
- * @param capFactor Capacity factor.
- */
- private void printPageIdDistribution(
- int cacheId,
- int parts,
- int pagesPerPartition,
- int segments,
- float capFactor
- ) {
- int allIds = parts * pagesPerPartition;
-
- int perSegmentSize = allIds / segments;
- int capacity = (int)(perSegmentSize * capFactor);
-
- info("Total ids: " + allIds);
-
- List<Map<Integer, Integer>> collisionsPerSegment = new ArrayList<>(segments);
-
- for (int i = 0; i < segments; i++)
- collisionsPerSegment.add(new HashMap<Integer, Integer>(allIds / segments, 1.0f));
-
- int[] numInSegment = new int[segments];
-
- for (int p = 0; p < parts; p++) {
- for (int i = 0; i < pagesPerPartition; i++) {
- long pageId = PageIdUtils.pageId(p, (byte)0, i);
-
- int segment = PageMemoryImpl.segmentIndex(cacheId, pageId, segments);
-
- int idxInSegment = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
-
- Map<Integer, Integer> idxCollisions = collisionsPerSegment.get(segment);
-
- Integer old = idxCollisions.get(idxInSegment);
- idxCollisions.put(idxInSegment, old == null ? 1 : old + 1);
-
- numInSegment[segment]++;
- }
- }
-
- for (int i = 0; i < collisionsPerSegment.size(); i++) {
- Map<Integer, Integer> idxCollisions = collisionsPerSegment.get(i);
-
- int distinctPositions = idxCollisions.size();
-
- int totalCnt = 0;
- int nonZero = 0;
-
- for (Map.Entry<Integer, Integer> collision : idxCollisions.entrySet()) {
- if (collision.getValue() != null) {
- totalCnt += collision.getValue();
- nonZero++;
- }
- }
-
- info(String.format("Segment stats [i=%d, total=%d, distinct=%d, spaceUsed=%d%%, avgItCnt=%.1f + ']",
- i, numInSegment[i], distinctPositions, distinctPositions * 100 / numInSegment[i],
- (float)totalCnt / nonZero));
- }
-
- info("==========================================================");
- }
-
- /**
- * Uncomment and run this test manually to get data to plot histogram for per-element distance from ideal.
- * You can use Octave to plot the histogram:
- * <pre>
- * all = csvread("histo.txt");
- * hist(all, 200)
- * </pre>
- *
- * @throws Exception If failed.
- */
- public void _testRealHistory() throws Exception {
- int capacity = CACHE_IDS.length * PARTS * PAGES;
-
- info("Capacity: " + capacity);
-
- long mem = FullPageIdTable.requiredMemory(capacity);
-
- info(U.readableSize(mem, true));
-
- UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
-
- prov.initialize(new long[] {mem});
-
- DirectMemoryRegion region = prov.nextRegion();
-
- try {
- long seed = U.currentTimeMillis();
-
- info("Seed: " + seed + "L; //");
-
- Random rnd = new Random(seed);
-
- FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
-
- Map<T2<Integer, Integer>, Integer> allocated = new HashMap<>();
-
- for (int i = 0; i < capacity; i++) {
- int cacheId = CACHE_IDS[rnd.nextInt(CACHE_IDS.length)];
- int partId = rnd.nextInt(PARTS);
-
- T2<Integer, Integer> key = new T2<>(cacheId, partId);
-
- Integer pageIdx = allocated.get(key);
-
- pageIdx = pageIdx == null ? 1 : pageIdx + 1;
-
- if (pageIdx > PAGES)
- continue;
-
- tbl.put(cacheId, PageIdUtils.pageId(partId, (byte)0, pageIdx), 1, 0);
-
- allocated.put(key, pageIdx);
-
- if (i > 0 && i % 100_000 == 0)
- info("Done: " + i);
- }
-
- int[] scans = new int[capacity];
-
- int cur = 0;
-
- for (T2<Integer, Integer> key : allocated.keySet()) {
- Integer alloc = allocated.get(key);
-
- if (alloc != null) {
- for (int idx = 1; idx <= alloc; idx++) {
- scans[cur] = tbl.distanceFromIdeal(key.get1(), PageIdUtils.pageId(key.get2(), (byte)0, idx), 0);
-
- assert scans[cur] != -1;
-
- cur++;
- }
- }
- }
-
- try (FileOutputStream out = new FileOutputStream("histo.txt")) {
- PrintWriter w = new PrintWriter(new OutputStreamWriter(out));
-
- for (int scan : scans) {
- if (scan != 0)
- w.println(scan);
- }
-
- w.flush();
- }
- }
- finally {
- prov.shutdown();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplNoLoadTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplNoLoadTest.java
deleted file mode 100644
index 38baea8..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplNoLoadTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.io.File;
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoLoadSelfTest;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-
-/**
- *
- */
-public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
- /**
- * @return Page memory implementation.
- */
- @Override protected PageMemory memory() throws Exception {
- File memDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
-
- long[] sizes = new long[10];
-
- for (int i = 0; i < sizes.length; i++)
- sizes[i] = 5 * 1024 * 1024;
-
- DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), memDir);
-
- GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
- new GridTestKernalContext(log),
- null,
- null,
- null,
- new NoOpPageStoreManager(),
- new NoOpWALManager(),
- new IgniteCacheDatabaseSharedManager(),
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null
- );
-
- return new PageMemoryImpl(
- provider,
- sizes,
- sharedCtx,
- PAGE_SIZE,
- new CIX3<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuffer, Integer tag) {
- assert false : "No evictions should happen during the test";
- }
- },
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
- }
- },
- new CheckpointLockStateChecker() {
- @Override public boolean checkpointLockIsHeldByThread() {
- return true;
- }
- },
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
- }
-
- /** {@inheritDoc} */
- @Override public void testPageHandleDeallocation() throws Exception {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplTest.java
deleted file mode 100644
index b5e4549..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/pagemem/PageMemoryImplTest.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.pagemem;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
-import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CheckpointLockStateChecker;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.typedef.CIX3;
-import org.apache.ignite.testframework.junits.GridTestKernalContext;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger;
-
-/**
- *
- */
-public class PageMemoryImplTest extends GridCommonAbstractTest {
- /** Mb. */
- private static final long MB = 1024 * 1024;
-
- /** Page size. */
- private static final int PAGE_SIZE = 1024;
-
- /**
- * @throws Exception if failed.
- */
- public void testThatAllocationTooMuchPagesCauseToOOMException() throws Exception {
- PageMemoryImpl memory = createPageMemory();
-
- try {
- while (!Thread.currentThread().isInterrupted())
- memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
- }
- catch (IgniteOutOfMemoryException ignore) {
- //Success
- }
-
- assertFalse(memory.safeToUpdate());
- }
-
- /**
- *
- */
- private PageMemoryImpl createPageMemory() throws Exception {
- long[] sizes = new long[5];
-
- for (int i = 0; i < sizes.length; i++)
- sizes[i] = 1024 * MB / 4;
-
- sizes[4] = 10 * MB;
-
- DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
-
- GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
- new GridTestKernalContext(new GridTestLog4jLogger()),
- null,
- null,
- null,
- new NoOpPageStoreManager(),
- new NoOpWALManager(),
- new IgniteCacheDatabaseSharedManager(),
- null,
- null,
- null,
- null,
- null,
- null,
- null,
- null
- );
-
- PageMemoryImpl mem = new PageMemoryImpl(
- provider,
- sizes,
- sharedCtx,
- PAGE_SIZE,
- new CIX3<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
- assert false : "No evictions should happen during the test";
- }
- },
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
- }
- }, new CheckpointLockStateChecker() {
- @Override public boolean checkpointLockIsHeldByThread() {
- return true;
- }
- },
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
-
- mem.start();
-
- return mem;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/AbstractNodeJoinTemplate.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/AbstractNodeJoinTemplate.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/AbstractNodeJoinTemplate.java
deleted file mode 100644
index 112ea03..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/AbstractNodeJoinTemplate.java
+++ /dev/null
@@ -1,743 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.util.typedef.CI1;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteCallable;
-import org.apache.ignite.lang.IgniteClosure;
-import org.apache.ignite.lang.IgniteInClosure;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Assert;
-
-import static org.apache.ignite.internal.util.IgniteUtils.field;
-
-/**
- *
- */
-public abstract class AbstractNodeJoinTemplate extends GridCommonAbstractTest {
- /** Cache 1. */
- protected static final String cache1 = "cache1";
-
- /** Cache 2. */
- protected static final String cache2 = "cache2";
-
- //Todo Cache with node filter.
- protected static final String cache3 = "cache3";
-
- protected static final String cache4 = "cache4";
-
- protected static final String cache5 = "cache5";
-
- /** Caches info. */
- public static final String CACHES_INFO = "cachesInfo";
-
- /** Registered caches. */
- public static final String REGISTERED_CACHES = "registeredCaches";
-
- /** Caches. */
- public static final String CACHES = "caches";
-
- /**
- * @param ig Ig.
- */
- protected static Map<String, DynamicCacheDescriptor> cacheDescriptors(IgniteEx ig) {
- return field(field(ig.context().cache(), CACHES_INFO), REGISTERED_CACHES);
- }
-
- /**
- * @param ig Ig.
- */
- protected static Map<String, GridCacheAdapter> caches(IgniteEx ig){
- return field(ig.context().cache(), CACHES);
- }
-
- /**
- *
- */
- public abstract JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception;
-
- /**
- *
- */
- public abstract JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception;
-
- /**
- *
- */
- public abstract JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception;
-
- /**
- *
- */
- public abstract JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception;
-
- /**
- *
- */
- public abstract JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception;
-
- // Client node join.
-
- public abstract JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception;
-
- public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception;
-
- public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception;
-
- public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception;
-
- public abstract JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception;
-
- /**
- *
- */
- public abstract void testJoinWithOutConfiguration() throws Exception;
-
- /**
- *
- */
- public abstract void testStaticCacheConfigurationOnJoin() throws Exception;
-
- /**
- *
- */
- public abstract void testStaticCacheConfigurationInCluster() throws Exception;
-
- /**
- *
- */
- public abstract void testStaticCacheConfigurationSameOnBoth() throws Exception;
-
- /**
- *
- */
- public abstract void testStaticCacheConfigurationDifferentOnBoth() throws Exception;
-
- /**
- *
- */
- public abstract void testJoinClientWithOutConfiguration() throws Exception;
-
- /**
- *
- */
- public abstract void testJoinClientStaticCacheConfigurationOnJoin() throws Exception;
-
- /**
- *
- */
- public abstract void testJoinClientStaticCacheConfigurationInCluster() throws Exception;
-
- /**
- *
- */
- public abstract void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception;
-
- /**
- *
- */
- public abstract void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception;
-
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @param idx Index.
- */
- protected String name(int idx) {
- return getTestIgniteInstanceName(idx);
- }
-
- /**
- * @param name Name.
- */
- protected IgniteConfiguration cfg(String name) throws Exception {
- try {
- return getConfiguration(name);
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- /**
- *
- */
- protected JoinNodeTestPlanBuilder builder() {
- return JoinNodeTestPlanBuilder.builder();
- }
-
- /**
- * @param cfgs Cfgs.
- */
- protected static <T> T[] buildConfiguration(T... cfgs) {
- return cfgs;
- }
-
- /**
- *
- */
- protected CacheConfiguration atomicCfg() {
- return new CacheConfiguration(cache1)
- .setAtomicityMode(CacheAtomicityMode.ATOMIC);
- }
-
- /**
- *
- */
- protected CacheConfiguration transactionCfg() {
- return new CacheConfiguration(cache2)
- .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- }
-
- /**
- *
- */
- protected CacheConfiguration[] allCacheConfigurations() {
- return buildConfiguration(atomicCfg(), transactionCfg());
- }
-
- /** Set client. */
- protected final IgniteClosure<IgniteConfiguration, IgniteConfiguration> setClient =
- new IgniteClosure<IgniteConfiguration, IgniteConfiguration>() {
- @Override public IgniteConfiguration apply(IgniteConfiguration cfg) {
- return cfg.setClientMode(true);
- }
- };
-
- /** Ip finder. */
- private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
- return super.getConfiguration(name)
- .setDiscoverySpi(
- new TcpDiscoverySpi()
- .setIpFinder(ipFinder)
- );
- }
-
- /** {@inheritDoc} */
- protected IgniteConfiguration persistentCfg(IgniteConfiguration cfg) throws Exception {
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
-
- return cfg;
- }
-
- /**
- *
- */
- public static class JoinNodeTestPlanBuilder extends GridCommonAbstractTest {
- /** String plan builder. */
- private final StringBuilder strPlanBuilder = new StringBuilder().append("**** Execution plan ****\n");
-
- /** Nodes. */
- protected List<String> nodes = new ArrayList<>(4);
-
- /** Cluster config. */
- private IgniteConfiguration[] clusterCfg;
-
- /** Node config. */
- private IgniteConfiguration nodeCfg;
-
- /** State default. */
- private static final Boolean stateDefault = new Boolean(true);
-
- /** State. */
- private Boolean state = stateDefault;
-
- /** Noop. */
- private static final Runnable Noop = new Runnable() {
- @Override public void run() {
- }
- };
-
- /** After cluster started. */
- private Runnable afterClusterStarted = Noop;
-
- /** After node join. */
- private Runnable afterNodeJoin = Noop;
-
- /** After activate. */
- private Runnable afterActivate = Noop;
-
- /** After de activate. */
- private Runnable afterDeActivate = Noop;
-
- private IgniteCallable<List<CacheConfiguration>> dynamicCacheStart =
- new IgniteCallable<List<CacheConfiguration>>() {
- @Override public List<CacheConfiguration> call() throws Exception {
- return Arrays.asList(new CacheConfiguration(cache4), new CacheConfiguration(cache5));
- }
- };
-
- private IgniteCallable<List<String>> dynamicCacheStop =
- new IgniteCallable<List<String>>() {
- @Override public List<String> call() throws Exception {
- return Arrays.asList(cache4, cache5);
- }
- };
-
- private Runnable afterDynamicCacheStarted = Noop;
-
- private Runnable afterDynamicCacheStopped = Noop;
-
- /** End. */
- private Runnable end = Noop;
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder clusterConfiguration(IgniteConfiguration... cfgs) throws Exception {
- clusterCfg = cfgs;
-
- strPlanBuilder.append("Start cluster:\n");
-
- for (IgniteConfiguration cfg : cfgs) {
- strPlanBuilder.append("node: ")
- .append(cfg.getIgniteInstanceName())
- .append(" activeOnStart - ")
- .append(cfg.isActiveOnStart())
- .append("\n");
-
- CacheConfiguration[] ccfgs = cfg.getCacheConfiguration();
-
- if (ccfgs != null) {
- for (CacheConfiguration ccfg : ccfgs)
- strPlanBuilder.append(" cache - ")
- .append(ccfg.getName())
- .append("\n");
- }
- }
-
- return this;
- }
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder nodeConfiguration(IgniteConfiguration cfg) {
- nodeCfg = cfg;
-
- strPlanBuilder.append("Join node:\n")
- .append(cfg.getIgniteInstanceName())
- .append(cfg.isClientMode() != null && cfg.isClientMode() ? " (client)" : "")
- .append(" activeOnStart - ")
- .append(cfg.isActiveOnStart())
- .append("\n");
-
- CacheConfiguration[] ccfgs = cfg.getCacheConfiguration();
-
- if (ccfgs != null)
- for (CacheConfiguration ccfg : ccfgs)
- strPlanBuilder.append(" cache - ").append(ccfg.getName()).append("\n");
-
- return this;
- }
-
- /**
- * @param func Func.
- */
- public JoinNodeTestPlanBuilder nodeConfiguration(
- IgniteClosure<IgniteConfiguration, IgniteConfiguration> func
- ) {
-
- nodeCfg = func.apply(nodeCfg);
-
- return this;
- }
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder afterClusterStarted(Runnable r) {
- strPlanBuilder.append("Check after cluster start\n");
-
- afterClusterStarted = r;
-
- return this;
- }
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder afterNodeJoin(Runnable r) {
- strPlanBuilder.append("Check after node join")
- .append("\n");
-
- afterNodeJoin = r;
-
- return this;
- }
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder stateAfterJoin(boolean state) {
- strPlanBuilder.append("Check state on all nodes after join, must be ")
- .append(state ? "<<active>>" : "<<inactive>>")
- .append(" \n");
-
- this.state = state;
-
- return this;
- }
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder afterActivate(Runnable r) {
- strPlanBuilder.append("Check after activate")
- .append("\n");
-
- afterActivate = r;
-
- return this;
- }
-
- /**
- *
- */
- public JoinNodeTestPlanBuilder afterDeActivate(Runnable r) {
- strPlanBuilder.append("Check after deActivate")
- .append("\n");
-
- afterDeActivate = r;
-
- return this;
- }
-
- public JoinNodeTestPlanBuilder dynamicCacheStart(IgniteCallable<List<CacheConfiguration>> caches){
- strPlanBuilder.append("Dynamic caches start")
- .append("\n");
-
- dynamicCacheStart = caches;
-
- return this;
- }
-
- public JoinNodeTestPlanBuilder afterDynamicCacheStarted(Runnable r){
- strPlanBuilder.append("Check after dynamic caches start")
- .append("\n");
-
- afterDynamicCacheStarted = r;
-
- return this;
- }
-
- public JoinNodeTestPlanBuilder dynamicCacheStop(IgniteCallable<List<String>> caches){
- strPlanBuilder.append("Dynamic caches stop")
- .append("\n");
-
- dynamicCacheStop = caches;
-
- return this;
- }
-
- public JoinNodeTestPlanBuilder afterDynamicCacheStopped(Runnable r){
- strPlanBuilder.append("Check after dynamic caches stop")
- .append("\n");
-
- afterDynamicCacheStopped = r;
-
- return this;
- }
-
- /**
- * @param end End.
- */
- public JoinNodeTestPlanBuilder setEnd(Runnable end) {
- strPlanBuilder.append("Check before stop")
- .append("\n");
-
- this.end = end;
-
- return this;
- }
-
- /**
- *
- */
- public void execute() throws Exception {
- try {
- if (state == stateDefault)
- fail("State after join must be specific. See JoinNodeTestPlanBuilder.stateAfterJoin(boolean).");
-
- System.out.println(strPlanBuilder.append("********************").toString());
-
- IgniteConfiguration[] cfgs = clusterCfg;
-
- System.out.println(">>> Start cluster");
-
- for (IgniteConfiguration cfg : cfgs) {
- startGrid(cfg);
-
- nodes.add(cfg.getIgniteInstanceName());
- }
-
- System.out.println(">>> Check after cluster started");
-
- afterClusterStarted.run();
-
- System.out.println(">>> Start new node");
-
- startGrid(nodeCfg);
-
- nodes.add(nodeCfg.getIgniteInstanceName());
-
- System.out.println(">>> Check after new node join in cluster");
-
- afterNodeJoin.run();
-
- System.out.println(">>> Check cluster state on all nodes");
-
- IgniteEx crd = grid(nodes.get(0));
-
- for (IgniteEx ig : grids())
- assertEquals((boolean)state, ig.active());
-
- if (!state) {
- System.out.println(">>> Activate cluster");
-
- crd.active(true);
-
- System.out.println(">>> Check after cluster activated");
-
- afterActivate.run();
- }
- else {
- System.out.println(">>> DeActivate cluster");
-
- crd.active(false);
-
- System.out.println(">>> Check after cluster deActivated");
-
- afterDeActivate.run();
-
- System.out.println(">>> Activate cluster");
-
- crd.active(true);
- }
-
- AffinityTopologyVersion next0Ver = nextMinorVersion(crd);
-
- crd.createCaches(dynamicCacheStart.call());
-
- awaitTopologyVersion(next0Ver);
-
- afterDynamicCacheStarted.run();
-
- onAllNode(new CI1<IgniteEx>() {
- @Override public void apply(IgniteEx ig) {
- if (ig.context().discovery().localNode().isClient())
- return;
-
- Assert.assertNotNull(ig.context().cache().cache(cache4));
- Assert.assertNotNull(ig.context().cache().cache(cache5));
-
- }
- });
-
- AffinityTopologyVersion next1Ver = nextMinorVersion(crd);
-
- crd.destroyCaches(dynamicCacheStop.call());
-
- afterDynamicCacheStopped.run();
-
- awaitTopologyVersion(next1Ver);
-
- onAllNode(new CI1<IgniteEx>() {
- @Override public void apply(IgniteEx ig) {
- if (ig.context().discovery().localNode().isClient())
- return;
-
- Assert.assertNull(ig.context().cache().cache(cache4));
- Assert.assertNull(ig.context().cache().cache(cache5));
-
- }
- });
-
- System.out.println(">>> Finish check");
-
- end.run();
- }
- finally {
- stopAllGrids();
- }
- }
-
- private AffinityTopologyVersion nextMinorVersion(IgniteEx ig){
- AffinityTopologyVersion cur = ig.context().discovery().topologyVersionEx();
-
- return new AffinityTopologyVersion(cur.topologyVersion(), cur.minorTopologyVersion() + 1);
- }
-
- private void awaitTopologyVersion(final AffinityTopologyVersion ver){
- onAllNode(new CI1<IgniteEx>() {
- @Override public void apply(IgniteEx ig) {
- while (true) {
- AffinityTopologyVersion locTopVer = ig.context().cache().context()
- .exchange().readyAffinityVersion();
-
- if (locTopVer.compareTo(ver) < 0){
- System.out.println("Top ready " + locTopVer + " on " + ig.localNode().id());
-
- try {
- Thread.sleep(100);
- }
- catch (InterruptedException e) {
- break;
- }
- }
- else
- break;
- }
- }
- }).run();
-
- }
-
- /**
- *
- */
- protected List<IgniteEx> grids() {
- List<IgniteEx> res = new ArrayList<>();
-
- for (String name : nodes)
- res.add(grid(name));
-
- return res;
- }
-
- /**
- *
- */
- public static JoinNodeTestPlanBuilder builder() {
- return new JoinNodeTestPlanBuilder();
- }
-
- /**
- *
- */
- public Runnable checkCacheOnlySystem() {
- return onAllNode(new IgniteInClosure<IgniteEx>() {
- @Override public void apply(IgniteEx ig) {
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(2, desc.size());
-
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(2, caches.size());
- }
- });
- }
-
- /**
- *
- */
- public Runnable checkCacheEmpty() {
- return onAllNode(new IgniteInClosure<IgniteEx>() {
- @Override public void apply(IgniteEx ig) {
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertTrue(desc.isEmpty());
-
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(0, caches.size());
- }
- });
- }
-
- /**
- *
- */
- public Runnable checkCacheNotEmpty() {
- return onAllNode(new IgniteInClosure<IgniteEx>() {
- @Override public void apply(IgniteEx ig) {
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- });
- }
-
- /**
- * @param cls Closure.
- */
- private Runnable onAllNode(final IgniteInClosure<IgniteEx> cls) {
- return new Runnable() {
- @Override public void run() {
- for (IgniteEx ig : grids()) {
- try {
- cls.apply(ig);
- }
- catch (AssertionError e) {
- System.out.println("Assertion on " + ig.name());
-
- throw e;
- }
- }
- }
- };
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateAbstractTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateAbstractTest.java
deleted file mode 100644
index 5bc5c3a..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateAbstractTest.java
+++ /dev/null
@@ -1,366 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public abstract class IgniteChangeGlobalStateAbstractTest extends GridCommonAbstractTest {
- /** Primary suffix. */
- private static final String primarySuffix = "-primary";
-
- /** BackUp suffix. */
- private static final String backUpSuffix = "-backUp";
-
- /** BackUp suffix. */
- private static final String clientSuffix = "-client";
-
- /** Primary ip finder. */
- protected final TcpDiscoveryIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Back up ip finder. */
- protected final TcpDiscoveryIpFinder backUpIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Consistent id count. */
- private int consistentIdCnt;
-
- /** Nodes. */
- protected Map<String, Ignite> nodes = new ConcurrentHashMap<>();
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- nodes.clear();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName(), true));
-
- startPrimaryNodes(primaryNodes());
-
- startPrimaryClientNodes(primaryClientNodes());
-
- startBackUpNodes(backUpNodes());
-
- startBackUpClientNodes(backUpClientNodes());
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- stopAll(clientSuffix);
-
- stopAll(primarySuffix);
-
- stopAll(backUpSuffix);
-
- nodes.clear();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName(), true));
- }
-
- /**
- *
- */
- protected int primaryNodes() {
- return 3;
- }
-
- /**
- *
- */
- protected int primaryClientNodes() {
- return 3;
- }
-
- /**
- *
- */
- protected int backUpNodes() {
- return 3;
- }
-
- /**
- *
- */
- protected int backUpClientNodes() {
- return 3;
- }
-
- /**
- * @param idx idx.
- */
- protected Ignite primary(int idx) {
- return nodes.get("node" + idx + primarySuffix);
- }
-
- /**
- * @param idx idx.
- */
- protected Ignite primaryClient(int idx) {
- return nodes.get("node" + idx + primarySuffix + clientSuffix);
- }
-
- /**
- * @param idx idx.
- */
- protected Ignite backUp(int idx) {
- return nodes.get("node" + idx + backUpSuffix);
- }
-
- /**
- * @param idx idx.
- */
- protected Ignite backUpClient(int idx) {
- return nodes.get("node" + idx + backUpSuffix + clientSuffix);
- }
-
- /**
- * @param cnt Count.
- */
- protected void startPrimaryNodes(int cnt) throws Exception {
- for (int i = 0; i < cnt; i++)
- startPrimary(i);
- }
-
- /**
- * @param idx Index.
- */
- protected void startPrimary(int idx) throws Exception {
- String node = "node" + idx;
-
- String name = node + primarySuffix;
-
- IgniteConfiguration cfg = getConfiguration(name);
- cfg.setConsistentId(node);
- cfg.setActiveOnStart(true);
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder);
-
- Ignite ig = startGrid(name, cfg);
-
- nodes.put(name, ig);
- }
-
- /**
- * @param cnt Count.
- */
- protected void startBackUpNodes(int cnt) throws Exception {
- for (int i = 0; i < cnt; i++)
- startBackUp(i);
- }
-
- /**
- * @param idx Index.
- */
- protected void startBackUp(int idx) throws Exception {
- String node = "node" + idx;
-
- String name = node + backUpSuffix;
-
- IgniteConfiguration cfg = getConfiguration(name);
- cfg.setConsistentId(node);
- cfg.setActiveOnStart(false);
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
-
- Ignite ig = startGrid(name, cfg);
-
- nodes.put(name, ig);
- }
-
- /**
- * @param cnt Count.
- */
- protected void startPrimaryClientNodes(int cnt) throws Exception {
- for (int i = 0; i < cnt; i++) {
- String node = "node" + i;
-
- String name = node + primarySuffix + clientSuffix;
-
- IgniteConfiguration cfg = getConfiguration(name);
- cfg.setConsistentId(node);
- cfg.setClientMode(true);
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder);
-
- Ignite ig = startGrid(name, cfg);
-
- nodes.put(name, ig);
- }
- }
-
- /**
- * @param cnt Count.
- */
- protected void startBackUpClientNodes(int cnt) throws Exception {
- for (int i = 0; i < cnt; i++) {
- String node = "node" + i;
-
- String name = node + backUpSuffix + clientSuffix;
-
- IgniteConfiguration cfg = getConfiguration(name);
- cfg.setConsistentId(node);
- cfg.setActiveOnStart(false);
- cfg.setClientMode(true);
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
-
- Ignite ig = startGrid(name, cfg);
-
- nodes.put(name, ig);
- }
- }
-
- /**
- *
- */
- protected Iterable<Ignite> allBackUpNodes(){
- List<Ignite> r = new ArrayList<>();
-
- for (String name : this.nodes.keySet())
- if (name.contains(backUpSuffix))
- r.add(nodes.get(name));
-
- return r;
- }
-
- /**
- *
- */
- protected Ignite randomBackUp(boolean includeClient) {
- int nodes = 0;
-
- List<Ignite> igs = new ArrayList<>();
-
- for (String name : this.nodes.keySet())
- if (name.contains(backUpSuffix)){
- if (includeClient)
- igs.add(this.nodes.get(name));
- else {
- if (name.contains(clientSuffix))
- continue;
-
- igs.add(this.nodes.get(name));
- }
- }
-
- int idx = ThreadLocalRandom.current().nextInt(0, igs.size());
-
- return igs.get(idx);
- }
-
-
- /**
- * @param i Idx.
- */
- protected void stopPrimary(int i) {
- String name = "node" + i + primarySuffix;
-
- nodes.get(name).close();
-
- nodes.remove(name);
- }
-
- /**
- *
- */
- protected void stopAllPrimary() {
- stopAll(primarySuffix);
- }
-
- /**
- *
- */
- protected void stopAllBackUp() {
- stopAll(backUpSuffix);
- }
-
- /**
- *
- */
- protected void stopAllClient() {
- stopAll(clientSuffix);
- }
-
- /**
- * @param suffix Suffix.
- */
- private void stopAll(String suffix) {
- for (String name : nodes.keySet())
- if (name.contains(suffix)) {
- Ignite ig = nodes.get(name);
-
- stopGrid(ig.name());
-
- nodes.remove(name);
- }
- }
-
- /**
- * @param gridName Grid name.
- */
- @Override protected IgniteConfiguration getConfiguration(final String gridName) throws Exception {
- final IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
-
- pCfg.setPersistentStorePath(testName() + "/db");
- pCfg.setWalArchivePath(testName() + "/db/wal/archive");
- pCfg.setWalStorePath(testName() + "/db/wal");
-
- cfg.setPersistentStoreConfiguration(pCfg);
-
- final MemoryConfiguration memCfg = new MemoryConfiguration();
-
- memCfg.setPageSize(1024);
- memCfg.setConcurrencyLevel(64);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
- memPlcCfg.setInitialSize(20 * 1024 * 1024);
- memPlcCfg.setMaxSize(20 * 1024 * 1024);
- memPlcCfg.setName("dfltMemPlc");
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- return cfg;
- }
-
- /**
- *
- */
- protected String testName() {
- return getClass().getSimpleName();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateCacheTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateCacheTest.java
deleted file mode 100644
index ebbab58..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateCacheTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import javax.cache.configuration.Configuration;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.util.typedef.F;
-
-/**
- *
- */
-public class IgniteChangeGlobalStateCacheTest extends IgniteChangeGlobalStateAbstractTest {
- /**
- *
- */
- public void testCheckValueAfterActivation(){
- String cacheName = "my-cache";
-
- Ignite ig1P = primary(0);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- IgniteCache<String, String> cacheP = ig1P.getOrCreateCache(cacheName);
-
- cacheP.put("key","value");
-
- stopAllPrimary();
-
- ig1B.active(true);
-
- IgniteCache<String, String> cache1B = ig1B.cache(cacheName);
- IgniteCache<String, String> cache2B = ig2B.cache(cacheName);
- IgniteCache<String, String> cache3B = ig3B.cache(cacheName);
-
- assertTrue(cache1B != null);
- assertTrue(cache2B != null);
- assertTrue(cache3B != null);
-
- assertEquals(cache1B.get("key"), "value");
- assertEquals(cache2B.get("key"), "value");
- assertEquals(cache3B.get("key"), "value");
- }
-
- /**
- *
- */
- public void testMoreKeyValueAfterActivate() throws Exception {
- String cacheName = "my-cache";
-
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- CacheConfiguration<String, String> cacheCfg = new CacheConfiguration<>(cacheName);
-
- IgniteCache<String, String> cache1P = ig1P.getOrCreateCache(cacheCfg);
-
- for (int i = 0; i < 4_000; i++)
- cache1P.put("key" + i, "value" + i);
-
- IgniteCache<String, String> cache2P = ig2P.cache(cacheName);
-
- for (int i = 4_000; i < 8_000; i++)
- cache2P.put("key" + i, "value" + i);
-
- IgniteCache<String, String> cache3P = ig3P.cache(cacheName);
-
- for (int i = 8_000; i < 12_000; i++)
- cache3P.put("key" + i, "value" + i);
-
- stopAllPrimary();
-
- ig1B.active(true);
-
- IgniteCache<String, String> cache1B = ig1B.cache(cacheName);
- IgniteCache<String, String> cache2B = ig2B.cache(cacheName);
- IgniteCache<String, String> cache3B = ig3B.cache(cacheName);
-
- assertTrue(cache1B != null);
- assertTrue(cache2B != null);
- assertTrue(cache3B != null);
-
- for (int i = 0; i < 4_000; i++)
- assertEquals("value" + i, cache1B.get("key" + i));
-
- for (int i = 4_000; i < 8_000; i++)
- assertEquals("value" + i, cache2B.get("key" + i));
-
- for (int i = 8_000; i < 12_000; i++)
- assertEquals("value" + i, cache3B.get("key" + i));
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testDeActivateAndActivateCacheValue() throws Exception {
- String chName = "myCache";
-
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- IgniteCache<String, String> cacheExp = ig1.getOrCreateCache(chName);
-
- Configuration<String, String> cfgExp = cacheExp.getConfiguration(CacheConfiguration.class);
-
- cacheExp.put("key", "value");
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig2.active(false);
-
- IgniteEx ex1 = (IgniteEx)ig1;
- IgniteEx ex2 = (IgniteEx)ig2;
- IgniteEx ex3 = (IgniteEx)ig3;
-
- GridCacheProcessor cache1 = ex1.context().cache();
- GridCacheProcessor cache2 = ex2.context().cache();
- GridCacheProcessor cache3 = ex3.context().cache();
-
- assertTrue(F.isEmpty(cache1.jcaches()));
- assertTrue(F.isEmpty(cache2.jcaches()));
- assertTrue(F.isEmpty(cache3.jcaches()));
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- IgniteCache<String, String> cacheAct = ig2.cache(chName);
-
- Configuration<String, String> cfgAct = cacheAct.getConfiguration(CacheConfiguration.class);
-
- assertEquals("value", cacheAct.get("key"));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java
deleted file mode 100644
index bd65597..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStreamerTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-
-/**
- *
- */
-public class IgniteChangeGlobalStateDataStreamerTest extends IgniteChangeGlobalStateAbstractTest {
- /** {@inheritDoc} */
- @Override protected int backUpNodes() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override protected int backUpClientNodes() {
- return 0;
- }
-
- /**
- *
- */
- public void testDeActivateAndActivateDataStreamer() throws InterruptedException {
-
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- Ignite ig1C = primaryClient(0);
- Ignite ig2C = primaryClient(1);
- Ignite ig3C = primaryClient(2);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- assertTrue(ig1C.active());
- assertTrue(ig2C.active());
- assertTrue(ig3C.active());
-
- String cacheName = "myStreamCache";
-
- ig2C.getOrCreateCache(cacheName);
-
- try (IgniteDataStreamer<Integer, String> stmr = ig1.dataStreamer(cacheName)) {
- for (int i = 0; i < 100; i++)
- stmr.addData(i, Integer.toString(i));
- }
-
- ig2C.active(false);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- boolean fail = false;
-
- try {
- IgniteDataStreamer<String, String> strm2 = ig2.dataStreamer(cacheName);
- }
- catch (Exception e) {
- fail = true;
-
- assertTrue(e.getMessage().contains("can not perform operation, because cluster inactive"));
- }
-
- if (!fail)
- fail("exception was not throw");
-
- ig3C.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- assertTrue(ig1C.active());
- assertTrue(ig2C.active());
- assertTrue(ig3C.active());
-
- try (IgniteDataStreamer<Integer, String> stmr2 = ig2.dataStreamer(cacheName)) {
- for (int i = 100; i < 200; i++)
- stmr2.addData(i, Integer.toString(i));
- }
-
- IgniteCache<Integer, String> cache = ig3.cache(cacheName);
-
- for (int i = 0; i < 200; i++)
- assertEquals(String.valueOf(i), cache.get(i));
- }
-}
[31/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusIO.java
new file mode 100644
index 0000000..4c2b2e0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusIO.java
@@ -0,0 +1,403 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.lang.IgniteInClosure;
+
+/**
+ * Abstract IO routines for B+Tree pages.
+ */
+public abstract class BPlusIO<L> extends PageIO {
+ /** */
+ private static final int CNT_OFF = COMMON_HEADER_END;
+
+ /** */
+ private static final int FORWARD_OFF = CNT_OFF + 2;
+
+ /** */
+ private static final int REMOVE_ID_OFF = FORWARD_OFF + 8;
+
+ /** */
+ protected static final int ITEMS_OFF = REMOVE_ID_OFF + 8;
+
+ /** */
+ private final boolean canGetRow;
+
+ /** */
+ private final boolean leaf;
+
+ /** All the items must be of fixed size. */
+ protected final int itemSize;
+
+ /**
+ * @param type Page type.
+ * @param ver Page format version.
+ * @param leaf If this is a leaf IO.
+ * @param canGetRow If we can get full row from this page.
+ */
+ protected BPlusIO(int type, int ver, boolean leaf, boolean canGetRow, int itemSize) {
+ super(type, ver);
+
+ assert itemSize > 0 : itemSize;
+ assert canGetRow || !leaf: "leaf page always must be able to get full row";
+
+ this.leaf = leaf;
+ this.canGetRow = canGetRow;
+ this.itemSize = itemSize;
+ }
+
+ /**
+ * @return Item size in bytes.
+ */
+ public final int getItemSize() {
+ return itemSize;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setCount(pageAddr, 0);
+ setForward(pageAddr, 0);
+ setRemoveId(pageAddr, 0);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Forward page ID.
+ */
+ public final long getForward(long pageAddr) {
+ return PageUtils.getLong(pageAddr, FORWARD_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageId Forward page ID.
+ */
+ public final void setForward(long pageAddr, long pageId) {
+ PageUtils.putLong(pageAddr, FORWARD_OFF, pageId);
+
+ assert getForward(pageAddr) == pageId;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Remove ID.
+ */
+ public final long getRemoveId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, REMOVE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param rmvId Remove ID.
+ */
+ public final void setRemoveId(long pageAddr, long rmvId) {
+ PageUtils.putLong(pageAddr, REMOVE_ID_OFF, rmvId);
+
+ assert getRemoveId(pageAddr) == rmvId;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Items count in the page.
+ */
+ public final int getCount(long pageAddr) {
+ int cnt = PageUtils.getShort(pageAddr, CNT_OFF) & 0xFFFF;
+
+ assert cnt >= 0: cnt;
+
+ return cnt;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Count.
+ */
+ public final void setCount(long pageAddr, int cnt) {
+ assert cnt >= 0: cnt;
+
+ PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
+
+ assert getCount(pageAddr) == cnt;
+ }
+
+ /**
+ * @return {@code true} If we can get the full row from this page using
+ * method {@link BPlusTree#getRow(BPlusIO, long, int)}.
+ * Must always be {@code true} for leaf pages.
+ */
+ public final boolean canGetRow() {
+ return canGetRow;
+ }
+
+ /**
+ * @return {@code true} if it is a leaf page.
+ */
+ public final boolean isLeaf() {
+ return leaf;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ * @return Max items count.
+ */
+ public abstract int getMaxCount(long pageAddr, int pageSize);
+
+ /**
+ * Store the needed info about the row in the page. Leaf and inner pages can store different info.
+ *
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param row Lookup or full row.
+ * @param rowBytes Row bytes.
+ * @param needRowBytes If we need stored row bytes.
+ * @return Stored row bytes.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final byte[] store(long pageAddr, int idx, L row, byte[] rowBytes, boolean needRowBytes)
+ throws IgniteCheckedException {
+ int off = offset(idx);
+
+ if (rowBytes == null) {
+ storeByOffset(pageAddr, off, row);
+
+ if (needRowBytes)
+ rowBytes = PageUtils.getBytes(pageAddr, off, getItemSize());
+ }
+ else
+ putBytes(pageAddr, off, rowBytes);
+
+ return rowBytes;
+ }
+
+ /**
+ * @param idx Index of element.
+ * @return Offset from byte buffer begin in bytes.
+ */
+ public abstract int offset(int idx);
+
+ /**
+ * Store the needed info about the row in the page. Leaf and inner pages can store different info.
+ *
+ * @param pageAddr Page address.
+ * @param off Offset in bytes.
+ * @param row Lookup or full row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void storeByOffset(long pageAddr, int off, L row) throws IgniteCheckedException;
+
+ /**
+ * Store row info from the given source.
+ *
+ * @param dstPageAddr Destination page address.
+ * @param dstIdx Destination index.
+ * @param srcIo Source IO.
+ * @param srcPageAddr Source page address.
+ * @param srcIdx Source index.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void store(long dstPageAddr, int dstIdx, BPlusIO<L> srcIo, long srcPageAddr, int srcIdx)
+ throws IgniteCheckedException;
+
+ /**
+ * Get lookup row.
+ *
+ * @param tree Tree.
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Lookup row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract L getLookupRow(BPlusTree<L, ?> tree, long pageAddr, int idx) throws IgniteCheckedException;
+
+ /**
+ * Copy items from source page to destination page.
+ * Both pages must be of the same type and the same version.
+ *
+ * @param srcPageAddr Source page address.
+ * @param dstPageAddr Destination page address.
+ * @param srcIdx Source begin index.
+ * @param dstIdx Destination begin index.
+ * @param cnt Items count.
+ * @param cpLeft Copy leftmost link (makes sense only for inner pages).
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract void copyItems(long srcPageAddr, long dstPageAddr, int srcIdx, int dstIdx, int cnt, boolean cpLeft)
+ throws IgniteCheckedException;
+
+ // Methods for B+Tree logic.
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param row Row to insert.
+ * @param rowBytes Row bytes.
+ * @param rightId Page ID which will be to the right child for the inserted item.
+ * @param needRowBytes If we need stored row bytes.
+ * @return Row bytes.
+ * @throws IgniteCheckedException If failed.
+ */
+ public byte[] insert(long pageAddr, int idx, L row, byte[] rowBytes, long rightId, boolean needRowBytes)
+ throws IgniteCheckedException {
+ int cnt = getCount(pageAddr);
+
+ // Move right all the greater elements to make a free slot for a new row link.
+ copyItems(pageAddr, pageAddr, idx, idx + 1, cnt - idx, false);
+
+ setCount(pageAddr, cnt + 1);
+
+ return store(pageAddr, idx, row, rowBytes, needRowBytes);
+ }
+
+ /**
+ * @param pageAddr Splitting page address.
+ * @param fwdId Forward page ID.
+ * @param fwdPageAddr Forward page address.
+ * @param mid Bisection index.
+ * @param cnt Initial elements count in the page being split.
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void splitForwardPage(
+ long pageAddr,
+ long fwdId,
+ long fwdPageAddr,
+ int mid,
+ int cnt,
+ int pageSize
+ ) throws IgniteCheckedException {
+ initNewPage(fwdPageAddr, fwdId, pageSize);
+
+ cnt -= mid;
+
+ copyItems(pageAddr, fwdPageAddr, mid, 0, cnt, true);
+
+ setCount(fwdPageAddr, cnt);
+ setForward(fwdPageAddr, getForward(pageAddr));
+
+ // Copy remove ID to make sure that if inner remove touched this page, then retry
+ // will happen even for newly allocated forward page.
+ setRemoveId(fwdPageAddr, getRemoveId(pageAddr));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param mid Bisection index.
+ * @param fwdId New forward page ID.
+ */
+ public void splitExistingPage(long pageAddr, int mid, long fwdId) {
+ setCount(pageAddr, mid);
+ setForward(pageAddr, fwdId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param cnt Count.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void remove(long pageAddr, int idx, int cnt) throws IgniteCheckedException {
+ cnt--;
+
+ copyItems(pageAddr, pageAddr, idx + 1, idx, cnt - idx, false);
+ setCount(pageAddr, cnt);
+ }
+
+ /**
+ * @param prntIo Parent IO.
+ * @param prntPageAddr Parent page address.
+ * @param prntIdx Split key index in parent.
+ * @param leftPageAddr Left page address.
+ * @param rightPageAddr Right page address.
+ * @param emptyBranch We are merging an empty branch.
+ * @param pageSize Page size.
+ * @return {@code false} If we were not able to merge.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean merge(
+ BPlusIO<L> prntIo,
+ long prntPageAddr,
+ int prntIdx,
+ long leftPageAddr,
+ long rightPageAddr,
+ boolean emptyBranch,
+ int pageSize
+ ) throws IgniteCheckedException {
+ int prntCnt = prntIo.getCount(prntPageAddr);
+ int leftCnt = getCount(leftPageAddr);
+ int rightCnt = getCount(rightPageAddr);
+
+ int newCnt = leftCnt + rightCnt;
+
+ // Need to move down split key in inner pages. For empty branch merge parent key will be just dropped.
+ if (!isLeaf() && !emptyBranch)
+ newCnt++;
+
+ if (newCnt > getMaxCount(leftPageAddr, pageSize)) {
+ assert !emptyBranch;
+
+ return false;
+ }
+
+ setCount(leftPageAddr, newCnt);
+
+ // Move down split key in inner pages.
+ if (!isLeaf() && !emptyBranch) {
+ assert prntIdx >= 0 && prntIdx < prntCnt: prntIdx; // It must be adjusted already.
+
+ // We can be sure that we have enough free space to store split key here,
+ // because we've done remove already and did not release child locks.
+ store(leftPageAddr, leftCnt, prntIo, prntPageAddr, prntIdx);
+
+ leftCnt++;
+ }
+
+ copyItems(rightPageAddr, leftPageAddr, 0, leftCnt, rightCnt, !emptyBranch);
+ setForward(leftPageAddr, getForward(rightPageAddr));
+
+ long rmvId = getRemoveId(rightPageAddr);
+
+ // Need to have maximum remove ID.
+ if (rmvId > getRemoveId(leftPageAddr))
+ setRemoveId(leftPageAddr, rmvId);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pos Position in page.
+ * @param bytes Bytes.
+ */
+ private static void putBytes(long pageAddr, int pos, byte[] bytes) {
+ PageUtils.putBytes(pageAddr, pos, bytes);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param c Closure.
+ */
+ public void visit(long pageAddr, IgniteInClosure<L> c) {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusInnerIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusInnerIO.java
new file mode 100644
index 0000000..88fc8c8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusInnerIO.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+
+/**
+ * Abstract IO routines for B+Tree inner pages.
+ */
+public abstract class BPlusInnerIO<L> extends BPlusIO<L> {
+ /** */
+ private static final int SHIFT_LEFT = ITEMS_OFF;
+
+ /** */
+ private static final int SHIFT_LINK = SHIFT_LEFT + 8;
+
+ /** */
+ private final int SHIFT_RIGHT = SHIFT_LINK + itemSize;
+
+ /**
+ * @param type Page type.
+ * @param ver Page format version.
+ * @param canGetRow If we can get full row from this page.
+ * @param itemSize Single item size on page.
+ */
+ protected BPlusInnerIO(int type, int ver, boolean canGetRow, int itemSize) {
+ super(type, ver, false, canGetRow, itemSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getMaxCount(long pageAddr, int pageSize) {
+ // The structure of the page is the following:
+ // |ITEMS_OFF|w|A|x|B|y|C|z|
+ // where capital letters are data items, lowercase letters are 8 byte page references.
+ return (pageSize - ITEMS_OFF - 8) / (getItemSize() + 8);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Page ID.
+ */
+ public final long getLeft(long pageAddr, int idx) {
+ return PageUtils.getLong(pageAddr, (offset0(idx, SHIFT_LEFT)));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param pageId Page ID.
+ */
+ public final void setLeft(long pageAddr, int idx, long pageId) {
+ PageUtils.putLong(pageAddr, offset0(idx, SHIFT_LEFT), pageId);
+
+ assert pageId == getLeft(pageAddr, idx);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @return Page ID.
+ */
+ public final long getRight(long pageAddr, int idx) {
+ return PageUtils.getLong(pageAddr, offset0(idx, SHIFT_RIGHT));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param pageId Page ID.
+ */
+ private void setRight(long pageAddr, int idx, long pageId) {
+ PageUtils.putLong(pageAddr, offset0(idx, SHIFT_RIGHT), pageId);
+
+ assert pageId == getRight(pageAddr, idx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public final void copyItems(long srcPageAddr, long dstPageAddr, int srcIdx, int dstIdx, int cnt,
+ boolean cpLeft) throws IgniteCheckedException {
+ assert srcIdx != dstIdx || srcPageAddr != dstPageAddr;
+
+ cnt *= getItemSize() + 8; // From items to bytes.
+
+ if (dstIdx > srcIdx) {
+ PageHandler.copyMemory(srcPageAddr, dstPageAddr, offset(srcIdx), offset(dstIdx), cnt);
+
+ if (cpLeft)
+ PageUtils.putLong(dstPageAddr, offset0(dstIdx, SHIFT_LEFT), PageUtils.getLong(srcPageAddr, (offset0(srcIdx, SHIFT_LEFT))));
+ }
+ else {
+ if (cpLeft)
+ PageUtils.putLong(dstPageAddr, offset0(dstIdx, SHIFT_LEFT), PageUtils.getLong(srcPageAddr, (offset0(srcIdx, SHIFT_LEFT))));
+
+ PageHandler.copyMemory(srcPageAddr, dstPageAddr, offset(srcIdx), offset(dstIdx), cnt);
+ }
+ }
+
+ /**
+ * @param idx Index of element.
+ * @param shift It can be either link itself or left or right page ID.
+ * @return Offset from byte buffer begin in bytes.
+ */
+ private int offset0(int idx, int shift) {
+ return shift + (8 + getItemSize()) * idx;
+ }
+
+ /** {@inheritDoc} */
+ @Override public final int offset(int idx) {
+ return offset0(idx, SHIFT_LINK);
+ }
+
+ // Methods for B+Tree logic.
+
+ /** {@inheritDoc} */
+ @Override public byte[] insert(
+ long pageAddr,
+ int idx,
+ L row,
+ byte[] rowBytes,
+ long rightId,
+ boolean needRowBytes
+ ) throws IgniteCheckedException {
+ rowBytes = super.insert(pageAddr, idx, row, rowBytes, rightId, needRowBytes);
+
+ // Setup reference to the right page on split.
+ setRight(pageAddr, idx, rightId);
+
+ return rowBytes;
+ }
+
+ /**
+ * @param newRootPageAddr New root page address.
+ * @param newRootId New root ID.
+ * @param leftChildId Left child ID.
+ * @param row Moved up row.
+ * @param rowBytes Bytes.
+ * @param rightChildId Right child ID.
+ * @param pageSize Page size.
+ * @param needRowBytes If we need row bytes back.
+ * @return Row bytes.
+ * @throws IgniteCheckedException If failed.
+ */
+ public byte[] initNewRoot(
+ long newRootPageAddr,
+ long newRootId,
+ long leftChildId,
+ L row,
+ byte[] rowBytes,
+ long rightChildId,
+ int pageSize,
+ boolean needRowBytes
+ ) throws IgniteCheckedException {
+ initNewPage(newRootPageAddr, newRootId, pageSize);
+
+ setCount(newRootPageAddr, 1);
+ setLeft(newRootPageAddr, 0, leftChildId);
+ rowBytes = store(newRootPageAddr, 0, row, rowBytes, needRowBytes);
+ setRight(newRootPageAddr, 0, rightChildId);
+
+ return rowBytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusLeafIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusLeafIO.java
new file mode 100644
index 0000000..931bf10
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusLeafIO.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+
+/**
+ * Abstract IO routines for B+Tree leaf pages.
+ */
+public abstract class BPlusLeafIO<L> extends BPlusIO<L> {
+ /**
+ * @param type Page type.
+ * @param ver Page format version.
+ * @param itemSize Single item size on page.
+ */
+ protected BPlusLeafIO(int type, int ver, int itemSize) {
+ super(type, ver, true, true, itemSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int getMaxCount(long pageAddr, int pageSize) {
+ return (pageSize - ITEMS_OFF) / getItemSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public final void copyItems(long srcPageAddr, long dstPageAddr, int srcIdx, int dstIdx, int cnt,
+ boolean cpLeft) throws IgniteCheckedException {
+ assert srcIdx != dstIdx || srcPageAddr != dstPageAddr;
+
+ PageHandler.copyMemory(srcPageAddr, dstPageAddr, offset(srcIdx), offset(dstIdx),
+ cnt * getItemSize());
+ }
+
+ /** {@inheritDoc} */
+ @Override public final int offset(int idx) {
+ assert idx >= 0: idx;
+
+ return ITEMS_OFF + idx * getItemSize();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java
new file mode 100644
index 0000000..6f430c1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/BPlusMetaIO.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.pagemem.PageUtils;
+
+/**
+ * IO routines for B+Tree meta pages.
+ */
+public class BPlusMetaIO extends PageIO {
+ /** */
+ public static final IOVersions<BPlusMetaIO> VERSIONS = new IOVersions<>(
+ new BPlusMetaIO(1), new BPlusMetaIO(2)
+ );
+
+ /** */
+ private static final int LVLS_OFF = COMMON_HEADER_END;
+
+ /** */
+ private final int refsOff;
+
+ /** */
+ private final int inlineSizeOff;
+
+ /**
+ * @param ver Page format version.
+ */
+ private BPlusMetaIO(int ver) {
+ super(T_BPLUS_META, ver);
+
+ switch (ver) {
+ case 1:
+ inlineSizeOff = -1;
+ refsOff = LVLS_OFF + 1;
+ break;
+
+ case 2:
+ inlineSizeOff = LVLS_OFF + 1;
+ refsOff = inlineSizeOff + 2;
+ break;
+
+ default:
+ throw new IgniteException("invalid IO version: " + ver);
+ }
+ }
+
+ /**
+ * @param pageAdrr Page address.
+ * @param rootId Root page ID.
+ * @param pageSize Page size.
+ */
+ public void initRoot(long pageAdrr, long rootId, int pageSize) {
+ setLevelsCount(pageAdrr, 1, pageSize);
+ setFirstPageId(pageAdrr, 0, rootId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Number of levels in this tree.
+ */
+ public int getLevelsCount(long pageAddr) {
+ return PageUtils.getByte(pageAddr, LVLS_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ * @return Max levels possible for this page size.
+ */
+ private int getMaxLevels(long pageAddr, int pageSize) {
+ return (pageSize - refsOff) / 8;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param lvls Number of levels in this tree.
+ * @param pageSize Page size.
+ */
+ private void setLevelsCount(long pageAddr, int lvls, int pageSize) {
+ assert lvls >= 0 && lvls <= getMaxLevels(pageAddr, pageSize) : lvls;
+
+ PageUtils.putByte(pageAddr, LVLS_OFF, (byte)lvls);
+
+ assert getLevelsCount(pageAddr) == lvls;
+ }
+
+ /**
+ * @param lvl Level.
+ * @return Offset for page reference.
+ */
+ private int offset(int lvl) {
+ return lvl * 8 + refsOff;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param lvl Level.
+ * @return First page ID at that level.
+ */
+ public long getFirstPageId(long pageAddr, int lvl) {
+ return PageUtils.getLong(pageAddr, offset(lvl));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param lvl Level.
+ * @param pageId Page ID.
+ */
+ private void setFirstPageId(long pageAddr, int lvl, long pageId) {
+ assert lvl >= 0 && lvl < getLevelsCount(pageAddr) : lvl;
+
+ PageUtils.putLong(pageAddr, offset(lvl), pageId);
+
+ assert getFirstPageId(pageAddr, lvl) == pageId;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Root level.
+ */
+ public int getRootLevel(long pageAddr) {
+ int lvls = getLevelsCount(pageAddr); // The highest level page is root.
+
+ assert lvls > 0 : lvls;
+
+ return lvls - 1;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param rootPageId New root page ID.
+ * @param pageSize Page size.
+ */
+ public void addRoot(long pageAddr, long rootPageId, int pageSize) {
+ int lvl = getLevelsCount(pageAddr);
+
+ setLevelsCount(pageAddr, lvl + 1, pageSize);
+ setFirstPageId(pageAddr, lvl, rootPageId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ */
+ public void cutRoot(long pageAddr, int pageSize) {
+ int lvl = getRootLevel(pageAddr);
+
+ setLevelsCount(pageAddr, lvl, pageSize); // Decrease tree height.
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param size Offset size.
+ */
+ public void setInlineSize(long pageAddr, int size) {
+ if (getVersion() > 1)
+ PageUtils.putShort(pageAddr, inlineSizeOff, (short)size);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public int getInlineSize(long pageAddr) {
+ return getVersion() > 1 ? PageUtils.getShort(pageAddr, inlineSizeOff) : 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java
new file mode 100644
index 0000000..fef5612
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/CacheVersionIO.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ * Utility to read and write {@link GridCacheVersion} instances.
+ */
+public class CacheVersionIO {
+ /** */
+ private static final byte NULL_PROTO_VER = 0;
+
+ /** */
+ private static final byte MAX_PROTO_VER = 1;
+
+ /** */
+ private static final int NULL_SIZE = 1;
+
+ /** Serialized size in bytes. */
+ private static final int SIZE_V1 = 17;
+
+ /**
+ * @param ver Version.
+ * @param allowNull Is {@code null} version allowed.
+ * @return Serialized size in bytes.
+ */
+ public static int size(GridCacheVersion ver, boolean allowNull) {
+ if (ver == null) {
+ if (allowNull)
+ return NULL_SIZE;
+
+ throw new IllegalStateException("Cache version is null");
+ }
+
+ return SIZE_V1;
+ }
+
+ /**
+ * @param buf Byte buffer.
+ * @param ver Version to write.
+ * @param allowNull Is {@code null} version allowed.
+ */
+ public static void write(ByteBuffer buf, GridCacheVersion ver, boolean allowNull) {
+ if (ver == null) {
+ if (allowNull)
+ buf.put(NULL_PROTO_VER);
+ else
+ throw new IllegalStateException("Cache version is null");
+ }
+ else {
+ byte protoVer = 1; // Version of serialization protocol.
+
+ buf.put(protoVer);
+ buf.putInt(ver.topologyVersion());
+ buf.putInt(ver.nodeOrderAndDrIdRaw());
+ buf.putLong(ver.order());
+ }
+ }
+
+ /**
+ * @param addr Write address.
+ * @param ver Version to write.
+ * @param allowNull Is {@code null} version allowed.
+ */
+ public static void write(long addr, GridCacheVersion ver, boolean allowNull) {
+ if (ver == null) {
+ if (allowNull)
+ PageUtils.putByte(addr, 0, NULL_PROTO_VER);
+ else
+ throw new IllegalStateException("Cache version is null");
+ }
+ else {
+ byte protoVer = 1; // Version of serialization protocol.
+
+ PageUtils.putByte(addr, 0, protoVer);
+ PageUtils.putInt(addr, 1, ver.topologyVersion());
+ PageUtils.putInt(addr, 5, ver.nodeOrderAndDrIdRaw());
+ PageUtils.putLong(addr, 9, ver.order());
+ }
+ }
+
+ /**
+ * @param protoVer Serialization protocol version.
+ * @param allowNull Is {@code null} version allowed.
+ * @return Protocol version.
+ * @throws IgniteCheckedException if failed.
+ */
+ private static byte checkProtocolVersion(byte protoVer, boolean allowNull) throws IgniteCheckedException {
+ if (protoVer >= NULL_PROTO_VER && protoVer <= MAX_PROTO_VER) {
+ if (protoVer == NULL_PROTO_VER && !allowNull)
+ throw new IllegalStateException("Cache version is null.");
+
+ return protoVer;
+ }
+
+ throw new IgniteCheckedException("Unsupported protocol version: " + protoVer);
+ }
+
+ /**
+ * Gets needed buffer size to read the whole version instance.
+ * Does not change buffer position.
+ *
+ * @param buf Buffer.
+ * @param allowNull Is {@code null} version allowed.
+ * @return Size of serialized version.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static int readSize(ByteBuffer buf, boolean allowNull) throws IgniteCheckedException {
+ byte protoVer = checkProtocolVersion(buf.get(buf.position()), allowNull);
+
+ switch (protoVer) {
+ case NULL_PROTO_VER:
+ return NULL_SIZE;
+
+ case 1:
+ return SIZE_V1;
+
+ default:
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Reads GridCacheVersion instance from the given buffer. Moves buffer's position by the number of used
+ * bytes.
+ *
+ * @param buf Byte buffer.
+ * @param allowNull Is {@code null} version allowed.
+ * @return Version.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static GridCacheVersion read(ByteBuffer buf, boolean allowNull) throws IgniteCheckedException {
+ byte protoVer = checkProtocolVersion(buf.get(), allowNull);
+
+ if (protoVer == NULL_PROTO_VER)
+ return null;
+
+ int topVer = buf.getInt();
+ int nodeOrderDrId = buf.getInt();
+ long order = buf.getLong();
+
+ return new GridCacheVersion(topVer, nodeOrderDrId, order);
+ }
+
+ /**
+ * Reads GridCacheVersion instance from the given address.
+ *
+ * @param pageAddr Page address.
+ * @param allowNull Is {@code null} version allowed.
+ * @return Version.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static GridCacheVersion read(long pageAddr, boolean allowNull) throws IgniteCheckedException {
+ byte protoVer = checkProtocolVersion(PageUtils.getByte(pageAddr, 0), allowNull);
+
+ if (protoVer == NULL_PROTO_VER)
+ return null;
+
+ int topVer = PageUtils.getInt(pageAddr, 1);
+ int nodeOrderDrId = PageUtils.getInt(pageAddr, 5);
+ long order = PageUtils.getLong(pageAddr, 9);
+
+ return new GridCacheVersion(topVer, nodeOrderDrId, order);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
new file mode 100644
index 0000000..d8ed1a6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
@@ -0,0 +1,1450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Data pages IO.
+ */
+public class DataPageIO extends PageIO {
+ /** */
+ public static final IOVersions<DataPageIO> VERSIONS = new IOVersions<>(
+ new DataPageIO(1)
+ );
+
+ /** */
+ private static final int SHOW_ITEM = 0b0001;
+
+ /** */
+ private static final int SHOW_PAYLOAD_LEN = 0b0010;
+
+ /** */
+ private static final int SHOW_LINK = 0b0100;
+
+ /** */
+ private static final int FREE_LIST_PAGE_ID_OFF = COMMON_HEADER_END;
+
+ /** */
+ private static final int FREE_SPACE_OFF = FREE_LIST_PAGE_ID_OFF + 8;
+
+ /** */
+ private static final int DIRECT_CNT_OFF = FREE_SPACE_OFF + 2;
+
+ /** */
+ private static final int INDIRECT_CNT_OFF = DIRECT_CNT_OFF + 1;
+
+ /** */
+ private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
+
+ /** */
+ private static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
+
+ /** */
+ private static final int ITEM_SIZE = 2;
+
+ /** */
+ private static final int PAYLOAD_LEN_SIZE = 2;
+
+ /** */
+ private static final int LINK_SIZE = 8;
+
+ /** */
+ private static final int FRAGMENTED_FLAG = 0b10000000_00000000;
+
+ /** */
+ public static final int MIN_DATA_PAGE_OVERHEAD = ITEMS_OFF + ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
+
+ /**
+ * @param ver Page format version.
+ */
+ protected DataPageIO(int ver) {
+ super(T_DATA, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setEmptyPage(pageAddr, pageSize);
+ setFreeListPageId(pageAddr, 0L);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ */
+ private void setEmptyPage(long pageAddr, int pageSize) {
+ setDirectCount(pageAddr, 0);
+ setIndirectCount(pageAddr, 0);
+ setFirstEntryOffset(pageAddr, pageSize, pageSize);
+ setRealFreeSpace(pageAddr, pageSize - ITEMS_OFF, pageSize);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param freeListPageId Free list page ID.
+ */
+ public void setFreeListPageId(long pageAddr, long freeListPageId) {
+ PageUtils.putLong(pageAddr, FREE_LIST_PAGE_ID_OFF, freeListPageId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Free list page ID.
+ */
+ public long getFreeListPageId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, FREE_LIST_PAGE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param show What elements of data page entry to show in the result.
+ * @return Data page entry size.
+ */
+ private int getPageEntrySize(long pageAddr, int dataOff, int show) {
+ int payloadLen = PageUtils.getShort(pageAddr, dataOff) & 0xFFFF;
+
+ if ((payloadLen & FRAGMENTED_FLAG) != 0)
+ payloadLen &= ~FRAGMENTED_FLAG; // We are fragmented and have a link.
+ else
+ show &= ~SHOW_LINK; // We are not fragmented, never have a link.
+
+ return getPageEntrySize(payloadLen, show);
+ }
+
+ /**
+ * @param payloadLen Length of the payload, may be a full data row or a row fragment length.
+ * @param show What elements of data page entry to show in the result.
+ * @return Data page entry size.
+ */
+ private int getPageEntrySize(int payloadLen, int show) {
+ assert payloadLen > 0 : payloadLen;
+
+ int res = payloadLen;
+
+ if ((show & SHOW_LINK) != 0)
+ res += LINK_SIZE;
+
+ if ((show & SHOW_ITEM) != 0)
+ res += ITEM_SIZE;
+
+ if ((show & SHOW_PAYLOAD_LEN) != 0)
+ res += PAYLOAD_LEN_SIZE;
+
+ return res;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Entry data offset.
+ * @param pageSize Page size.
+ */
+ private void setFirstEntryOffset(long pageAddr, int dataOff, int pageSize) {
+ assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff <= pageSize : dataOff;
+
+ PageUtils.putShort(pageAddr, FIRST_ENTRY_OFF, (short)dataOff);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Entry data offset.
+ */
+ private int getFirstEntryOffset(long pageAddr) {
+ return PageUtils.getShort(pageAddr, FIRST_ENTRY_OFF) & 0xFFFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param freeSpace Free space.
+ * @param pageSize Page size.
+ */
+ private void setRealFreeSpace(long pageAddr, int freeSpace, int pageSize) {
+ assert freeSpace == actualFreeSpace(pageAddr, pageSize) : freeSpace + " != " + actualFreeSpace(pageAddr, pageSize);
+
+ PageUtils.putShort(pageAddr, FREE_SPACE_OFF, (short)freeSpace);
+ }
+
+ /**
+ * Free space refers to a "max row size (without any data page specific overhead) which is
+ * guaranteed to fit into this data page".
+ *
+ * @param pageAddr Page address.
+ * @return Free space.
+ */
+ public int getFreeSpace(long pageAddr) {
+ if (getFreeItemSlots(pageAddr) == 0)
+ return 0;
+
+ int freeSpace = getRealFreeSpace(pageAddr);
+
+ // We reserve size here because of getFreeSpace() method semantics (see method javadoc).
+ // It means that we must be able to accommodate a row of size which is equal to getFreeSpace(),
+ // plus we will have data page overhead: header of the page as well as item, payload length and
+ // possibly a link to the next row fragment.
+ freeSpace -= ITEM_SIZE + PAYLOAD_LEN_SIZE + LINK_SIZE;
+
+ return freeSpace < 0 ? 0 : freeSpace;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return {@code true} If there is no useful data in this page.
+ */
+ public boolean isEmpty(long pageAddr) {
+ return getDirectCount(pageAddr) == 0;
+ }
+
+ /**
+ * Equivalent for {@link #actualFreeSpace(long, int)} but reads saved value.
+ *
+ * @param pageAddr Page address.
+ * @return Free space.
+ */
+ private int getRealFreeSpace(long pageAddr) {
+ return PageUtils.getShort(pageAddr, FREE_SPACE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Direct count.
+ */
+ private void setDirectCount(long pageAddr, int cnt) {
+ assert checkCount(cnt): cnt;
+
+ PageUtils.putByte(pageAddr, DIRECT_CNT_OFF, (byte)cnt);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Direct count.
+ */
+ private int getDirectCount(long pageAddr) {
+ return PageUtils.getByte(pageAddr, DIRECT_CNT_OFF) & 0xFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param c Closure.
+ * @param <T> Closure return type.
+ * @return Collection of closure results for all items in page.
+ * @throws IgniteCheckedException In case of error in closure body.
+ */
+ public <T> List<T> forAllItems(long pageAddr, CC<T> c) throws IgniteCheckedException {
+ long pageId = getPageId(pageAddr);
+
+ int cnt = getDirectCount(pageAddr);
+
+ List<T> res = new ArrayList<>(cnt);
+
+ for (int i = 0; i < cnt; i++) {
+ long link = PageIdUtils.link(pageId, i);
+
+ res.add(c.apply(link));
+ }
+
+ return res;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Indirect count.
+ */
+ private void setIndirectCount(long pageAddr, int cnt) {
+ assert checkCount(cnt): cnt;
+
+ PageUtils.putByte(pageAddr, INDIRECT_CNT_OFF, (byte)cnt);
+ }
+
+ /**
+ * @param idx Index.
+ * @return {@code true} If the index is valid.
+ */
+ private boolean checkIndex(int idx) {
+ return idx >= 0 && idx < 0xFF;
+ }
+
+ /**
+ * @param cnt Counter value.
+ * @return {@code true} If the counter fits 1 byte.
+ */
+ private boolean checkCount(int cnt) {
+ return cnt >= 0 && cnt <= 0xFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Indirect count.
+ */
+ private int getIndirectCount(long pageAddr) {
+ return PageUtils.getByte(pageAddr, INDIRECT_CNT_OFF) & 0xFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Number of free entry slots.
+ */
+ private int getFreeItemSlots(long pageAddr) {
+ return 0xFF - getDirectCount(pageAddr);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return Found index of indirect item.
+ */
+ private int findIndirectItemIndex(long pageAddr, int itemId, int directCnt, int indirectCnt) {
+ int low = directCnt;
+ int high = directCnt + indirectCnt - 1;
+
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+
+ int cmp = Integer.compare(itemId(getItem(pageAddr, mid)), itemId);
+
+ if (cmp < 0)
+ low = mid + 1;
+ else if (cmp > 0)
+ high = mid - 1;
+ else
+ return mid; // found
+ }
+
+ throw new IllegalStateException("Item not found: " + itemId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageSize Page size.
+ * @return String representation.
+ */
+ private String printPageLayout(long pageAddr, int pageSize) {
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+ int free = getRealFreeSpace(pageAddr);
+
+ boolean valid = directCnt >= indirectCnt;
+
+ SB b = new SB();
+
+ b.appendHex(PageIO.getPageId(pageAddr)).a(" [");
+
+ int entriesSize = 0;
+
+ for (int i = 0; i < directCnt; i++) {
+ if (i != 0)
+ b.a(", ");
+
+ short item = getItem(pageAddr, i);
+
+ if (item < ITEMS_OFF || item >= pageSize)
+ valid = false;
+
+ entriesSize += getPageEntrySize(pageAddr, item, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ b.a(item);
+ }
+
+ b.a("][");
+
+ Collection<Integer> set = new HashSet<>();
+
+ for (int i = directCnt; i < directCnt + indirectCnt; i++) {
+ if (i != directCnt)
+ b.a(", ");
+
+ short item = getItem(pageAddr, i);
+
+ int itemId = itemId(item);
+ int directIdx = directItemIndex(item);
+
+ if (!set.add(directIdx) || !set.add(itemId))
+ valid = false;
+
+ assert indirectItem(itemId, directIdx) == item;
+
+ if (itemId < directCnt || directIdx < 0 || directIdx >= directCnt)
+ valid = false;
+
+ if (i > directCnt && itemId(getItem(pageAddr, i - 1)) >= itemId)
+ valid = false;
+
+
+ b.a(itemId).a('^').a(directIdx);
+ }
+
+ b.a("][free=").a(free);
+
+ int actualFree = pageSize - ITEMS_OFF - (entriesSize + (directCnt + indirectCnt) * ITEM_SIZE);
+
+ if (free != actualFree) {
+ b.a(", actualFree=").a(actualFree);
+
+ valid = false;
+ }
+ else
+ b.a("]");
+
+ assert valid : b.toString();
+
+ return b.toString();
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param pageSize Page size.
+ * @return Data entry offset in bytes.
+ */
+ private int getDataOffset(long pageAddr, int itemId, int pageSize) {
+ assert checkIndex(itemId): itemId;
+
+ int directCnt = getDirectCount(pageAddr);
+
+ assert directCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", page=" + printPageLayout(pageAddr, pageSize);
+
+ if (itemId >= directCnt) { // Need to do indirect lookup.
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ // Must have indirect items here.
+ assert indirectCnt > 0: "itemId=" + itemId + ", directCnt=" + directCnt + ", indirectCnt=" + indirectCnt +
+ ", page=" + printPageLayout(pageAddr, pageSize);
+
+ int indirectItemIdx = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
+
+ assert indirectItemIdx >= directCnt : indirectItemIdx + " " + directCnt;
+ assert indirectItemIdx < directCnt + indirectCnt: indirectItemIdx + " " + directCnt + " " + indirectCnt;
+
+ itemId = directItemIndex(getItem(pageAddr, indirectItemIdx));
+
+ assert itemId >= 0 && itemId < directCnt: itemId + " " + directCnt + " " + indirectCnt; // Direct item.
+ }
+
+ return directItemToOffset(getItem(pageAddr, itemId));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Points to the entry start.
+ * @return Link to the next entry fragment or 0 if no fragments left or if entry is not fragmented.
+ */
+ private long getNextFragmentLink(long pageAddr, int dataOff) {
+ assert isFragmented(pageAddr, dataOff);
+
+ return PageUtils.getLong(pageAddr, dataOff + PAYLOAD_LEN_SIZE);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @return {@code true} If the data row is fragmented across multiple pages.
+ */
+ private boolean isFragmented(long pageAddr, int dataOff) {
+ return (PageUtils.getShort(pageAddr, dataOff) & FRAGMENTED_FLAG) != 0;
+ }
+
+ /**
+ * Sets position to start of actual fragment data and limit to it's end.
+ *
+ * @param pageAddr Page address.
+ * @param itemId Item to position on.
+ * @param pageSize Page size.
+ * @return {@link DataPagePayload} object.
+ */
+ public DataPagePayload readPayload(final long pageAddr, final int itemId, final int pageSize) {
+ int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+
+ boolean fragmented = isFragmented(pageAddr, dataOff);
+ long nextLink = fragmented ? getNextFragmentLink(pageAddr, dataOff) : 0;
+ int payloadSize = getPageEntrySize(pageAddr, dataOff, 0);
+
+ return new DataPagePayload(dataOff + PAYLOAD_LEN_SIZE + (fragmented ? LINK_SIZE : 0),
+ payloadSize,
+ nextLink);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Item index.
+ * @return Item.
+ */
+ private short getItem(long pageAddr, int idx) {
+ return PageUtils.getShort(pageAddr, itemOffset(idx));
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Item index.
+ * @param item Item.
+ */
+ private void setItem(long pageAddr, int idx, short item) {
+ PageUtils.putShort(pageAddr, itemOffset(idx), item);
+ }
+
+ /**
+ * @param idx Index of the item.
+ * @return Offset in buffer.
+ */
+ private int itemOffset(int idx) {
+ assert checkIndex(idx): idx;
+
+ return ITEMS_OFF + idx * ITEM_SIZE;
+ }
+
+ /**
+ * @param directItem Direct item.
+ * @return Offset of an entry payload inside of the page.
+ */
+ private int directItemToOffset(short directItem) {
+ return directItem & 0xFFFF;
+ }
+
+ /**
+ * @param dataOff Data offset.
+ * @return Direct item.
+ */
+ private short directItemFromOffset(int dataOff) {
+ assert dataOff >= ITEMS_OFF + ITEM_SIZE && dataOff < Short.MAX_VALUE: dataOff;
+
+ return (short)dataOff;
+ }
+
+ /**
+ * @param indirectItem Indirect item.
+ * @return Index of corresponding direct item.
+ */
+ private int directItemIndex(short indirectItem) {
+ return indirectItem & 0xFF;
+ }
+
+ /**
+ * @param indirectItem Indirect item.
+ * @return Fixed item ID (the index used for referencing an entry from the outside).
+ */
+ private int itemId(short indirectItem) {
+ return (indirectItem & 0xFFFF) >>> 8;
+ }
+
+ /**
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param directItemIdx Index of corresponding direct item.
+ * @return Indirect item.
+ */
+ private short indirectItem(int itemId, int directItemIdx) {
+ assert checkIndex(itemId): itemId;
+ assert checkIndex(directItemIdx): directItemIdx;
+
+ return (short)((itemId << 8) | directItemIdx);
+ }
+
+ /**
+ * Move the last direct item to the free slot and reference it with indirect item on the same place.
+ *
+ * @param pageAddr Page address.
+ * @param freeDirectIdx Free slot.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return {@code true} If the last direct item already had corresponding indirect item.
+ */
+ private boolean moveLastItem(long pageAddr, int freeDirectIdx, int directCnt, int indirectCnt) {
+ int lastIndirectId = findIndirectIndexForLastDirect(pageAddr, directCnt, indirectCnt);
+
+ int lastItemId = directCnt - 1;
+
+ assert lastItemId != freeDirectIdx;
+
+ short indirectItem = indirectItem(lastItemId, freeDirectIdx);
+
+ assert itemId(indirectItem) == lastItemId && directItemIndex(indirectItem) == freeDirectIdx;
+
+ setItem(pageAddr, freeDirectIdx, getItem(pageAddr, lastItemId));
+ setItem(pageAddr, lastItemId, indirectItem);
+
+ assert getItem(pageAddr, lastItemId) == indirectItem;
+
+ if (lastIndirectId != -1) { // Fix pointer to direct item.
+ setItem(pageAddr, lastIndirectId, indirectItem(itemId(getItem(pageAddr, lastIndirectId)), freeDirectIdx));
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return Index of indirect item for the last direct item.
+ */
+ private int findIndirectIndexForLastDirect(long pageAddr, int directCnt, int indirectCnt) {
+ int lastDirectId = directCnt - 1;
+
+ for (int i = directCnt, end = directCnt + indirectCnt; i < end; i++) {
+ short item = getItem(pageAddr, i);
+
+ if (directItemIndex(item) == lastDirectId)
+ return i;
+ }
+
+ return -1;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Item ID.
+ * @param pageSize Page size.
+ * @param payload Row data.
+ * @param row Row.
+ * @param rowSize Row size.
+ * @return {@code True} if entry is not fragmented.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean updateRow(
+ final long pageAddr,
+ int itemId,
+ int pageSize,
+ @Nullable byte[] payload,
+ @Nullable CacheDataRow row,
+ final int rowSize) throws IgniteCheckedException {
+ assert checkIndex(itemId) : itemId;
+ assert row != null ^ payload != null;
+
+ final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+
+ if (isFragmented(pageAddr, dataOff))
+ return false;
+
+ if (row != null)
+ writeRowData(pageAddr, dataOff, rowSize, row, false);
+ else
+ writeRowData(pageAddr, dataOff, payload);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param itemId Fixed item ID (the index used for referencing an entry from the outside).
+ * @param pageSize Page size.
+ * @return Next link for fragmented entries or {@code 0} if none.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long removeRow(long pageAddr, int itemId, int pageSize) throws IgniteCheckedException {
+ assert checkIndex(itemId) : itemId;
+
+ final int dataOff = getDataOffset(pageAddr, itemId, pageSize);
+ final long nextLink = isFragmented(pageAddr, dataOff) ? getNextFragmentLink(pageAddr, dataOff) : 0;
+
+ // Record original counts to calculate delta in free space in the end of remove.
+ final int directCnt = getDirectCount(pageAddr);
+ final int indirectCnt = getIndirectCount(pageAddr);
+
+ int curIndirectCnt = indirectCnt;
+
+ assert directCnt > 0 : directCnt; // Direct count always represents overall number of live items.
+
+ // Remove the last item on the page.
+ if (directCnt == 1) {
+ assert (indirectCnt == 0 && itemId == 0) ||
+ (indirectCnt == 1 && itemId == itemId(getItem(pageAddr, 1))) : itemId;
+
+ setEmptyPage(pageAddr, pageSize);
+ }
+ else {
+ // Get the entry size before the actual remove.
+ int rmvEntrySize = getPageEntrySize(pageAddr, dataOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ int indirectId = 0;
+
+ if (itemId >= directCnt) { // Need to remove indirect item.
+ assert indirectCnt > 0;
+
+ indirectId = findIndirectItemIndex(pageAddr, itemId, directCnt, indirectCnt);
+
+ assert indirectId >= directCnt;
+
+ itemId = directItemIndex(getItem(pageAddr, indirectId));
+
+ assert itemId < directCnt;
+ }
+
+ boolean dropLast = true;
+
+ if (itemId + 1 < directCnt) // It is not the last direct item.
+ dropLast = moveLastItem(pageAddr, itemId, directCnt, indirectCnt);
+
+ if (indirectId == 0) {// For the last direct item with no indirect item.
+ if (dropLast)
+ moveItems(pageAddr, directCnt, indirectCnt, -1, pageSize);
+ else
+ curIndirectCnt++;
+ }
+ else {
+ if (dropLast)
+ moveItems(pageAddr, directCnt, indirectId - directCnt, -1, pageSize);
+
+ moveItems(pageAddr, indirectId + 1, directCnt + indirectCnt - indirectId - 1, dropLast ? -2 : -1, pageSize);
+
+ if (dropLast)
+ curIndirectCnt--;
+ }
+
+ setIndirectCount(pageAddr, curIndirectCnt);
+ setDirectCount(pageAddr, directCnt - 1);
+
+ assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
+
+ // Increase free space.
+ setRealFreeSpace(pageAddr,
+ getRealFreeSpace(pageAddr) + rmvEntrySize + ITEM_SIZE * (directCnt - getDirectCount(pageAddr) + indirectCnt - getIndirectCount(pageAddr)),
+ pageSize);
+ }
+
+ return nextLink;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param idx Index.
+ * @param cnt Count.
+ * @param step Step.
+ * @param pageSize Page size.
+ */
+ private void moveItems(long pageAddr, int idx, int cnt, int step, int pageSize) {
+ assert cnt >= 0: cnt;
+
+ if (cnt != 0)
+ moveBytes(pageAddr, itemOffset(idx), cnt * ITEM_SIZE, step * ITEM_SIZE, pageSize);
+ }
+
+ /**
+ * @param newEntryFullSize New entry full size (with item, length and link).
+ * @param firstEntryOff First entry data offset.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @return {@code true} If there is enough space for the entry.
+ */
+ private boolean isEnoughSpace(int newEntryFullSize, int firstEntryOff, int directCnt, int indirectCnt) {
+ return ITEMS_OFF + ITEM_SIZE * (directCnt + indirectCnt) <= firstEntryOff - newEntryFullSize;
+ }
+
+ /**
+ * Adds row to this data page and sets respective link to the given row object.
+ *
+ * @param pageAddr Page address.
+ * @param row Cache data row.
+ * @param rowSize Row size.
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRow(
+ final long pageAddr,
+ CacheDataRow row,
+ final int rowSize,
+ final int pageSize
+ ) throws IgniteCheckedException {
+ assert rowSize <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
+
+ int fullEntrySize = getPageEntrySize(rowSize, SHOW_PAYLOAD_LEN | SHOW_ITEM);
+
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+ writeRowData(pageAddr, dataOff, rowSize, row, true);
+
+ int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+ setLink(row, pageAddr, itemId);
+ }
+
+ /**
+ * Adds row to this data page and sets respective link to the given row object.
+ *
+ * @param pageAddr Page address.
+ * @param payload Payload.
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRow(
+ long pageAddr,
+ byte[] payload,
+ int pageSize
+ ) throws IgniteCheckedException {
+ assert payload.length <= getFreeSpace(pageAddr): "can't call addRow if not enough space for the whole row";
+
+ int fullEntrySize = getPageEntrySize(payload.length, SHOW_PAYLOAD_LEN | SHOW_ITEM);
+
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+ writeRowData(pageAddr, dataOff, payload);
+
+ addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param entryFullSize New entry full size (with item, length and link).
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param dataOff First entry offset.
+ * @param pageSize Page size.
+ * @return First entry offset after compaction.
+ */
+ private int compactIfNeed(
+ final long pageAddr,
+ final int entryFullSize,
+ final int directCnt,
+ final int indirectCnt,
+ int dataOff,
+ int pageSize
+ ) {
+ if (!isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt)) {
+ dataOff = compactDataEntries(pageAddr, directCnt, pageSize);
+
+ assert isEnoughSpace(entryFullSize, dataOff, directCnt, indirectCnt);
+ }
+
+ return dataOff;
+ }
+
+ /**
+ * Put item reference on entry.
+ *
+ * @param pageAddr Page address.
+ * @param fullEntrySize Full entry size (with link, payload size and item).
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param dataOff Data offset.
+ * @param pageSize Page size.
+ * @return Item ID.
+ */
+ private int addItem(final long pageAddr,
+ final int fullEntrySize,
+ final int directCnt,
+ final int indirectCnt,
+ final int dataOff,
+ final int pageSize)
+ {
+ setFirstEntryOffset(pageAddr, dataOff, pageSize);
+
+ int itemId = insertItem(pageAddr, dataOff, directCnt, indirectCnt, pageSize);
+
+ assert checkIndex(itemId): itemId;
+ assert getIndirectCount(pageAddr) <= getDirectCount(pageAddr);
+
+ // Update free space. If number of indirect items changed, then we were able to reuse an item slot.
+ setRealFreeSpace(pageAddr,
+ getRealFreeSpace(pageAddr) - fullEntrySize + (getIndirectCount(pageAddr) != indirectCnt ? ITEM_SIZE : 0),
+ pageSize);
+
+ return itemId;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param fullEntrySize Full entry size.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param pageSize Page size.
+ * @return Offset in the buffer where the entry must be written.
+ */
+ private int getDataOffsetForWrite(long pageAddr, int fullEntrySize, int directCnt, int indirectCnt, int pageSize) {
+ int dataOff = getFirstEntryOffset(pageAddr);
+
+ // Compact if we do not have enough space for entry.
+ dataOff = compactIfNeed(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+ // We will write data right before the first entry.
+ dataOff -= fullEntrySize - ITEM_SIZE;
+
+ return dataOff;
+ }
+
+ /**
+ * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
+ *
+ * @param pageMem Page memory.
+ * @param pageAddr Page address.
+ * @param row Cache data row.
+ * @param written Number of bytes of row size that was already written.
+ * @param rowSize Row size.
+ * @param pageSize Page size.
+ * @return Written payload size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public int addRowFragment(
+ PageMemory pageMem,
+ long pageAddr,
+ CacheDataRow row,
+ int written,
+ int rowSize,
+ int pageSize
+ ) throws IgniteCheckedException {
+ return addRowFragment(pageMem, pageAddr, written, rowSize, row.link(), row, null, pageSize);
+ }
+
+ /**
+ * Adds this payload as a fragment to this data page.
+ *
+ * @param pageAddr Page address.
+ * @param payload Payload bytes.
+ * @param lastLink Link to the previous written fragment (link to the tail).
+ * @param pageSize Page size.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addRowFragment(
+ long pageAddr,
+ byte[] payload,
+ long lastLink,
+ int pageSize
+ ) throws IgniteCheckedException {
+ addRowFragment(null, pageAddr, 0, 0, lastLink, null, payload, pageSize);
+ }
+
+ /**
+ * Adds maximum possible fragment of the given row to this data page and sets respective link to the row.
+ *
+ * @param pageMem Page memory.
+ * @param pageAddr Page address.
+ * @param written Number of bytes of row size that was already written.
+ * @param rowSize Row size.
+ * @param lastLink Link to the previous written fragment (link to the tail).
+ * @param row Row.
+ * @param payload Payload bytes.
+ * @param pageSize Page size.
+ * @return Written payload size.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int addRowFragment(
+ PageMemory pageMem,
+ long pageAddr,
+ int written,
+ int rowSize,
+ long lastLink,
+ CacheDataRow row,
+ byte[] payload,
+ int pageSize
+ ) throws IgniteCheckedException {
+ assert payload == null ^ row == null;
+
+ int directCnt = getDirectCount(pageAddr);
+ int indirectCnt = getIndirectCount(pageAddr);
+
+ int payloadSize = payload != null ? payload.length :
+ Math.min(rowSize - written, getFreeSpace(pageAddr));
+
+ int fullEntrySize = getPageEntrySize(payloadSize, SHOW_PAYLOAD_LEN | SHOW_LINK | SHOW_ITEM);
+ int dataOff = getDataOffsetForWrite(pageAddr, fullEntrySize, directCnt, indirectCnt, pageSize);
+
+ if (payload == null) {
+ ByteBuffer buf = pageMem.pageBuffer(pageAddr);
+
+ buf.position(dataOff);
+
+ short p = (short)(payloadSize | FRAGMENTED_FLAG);
+
+ buf.putShort(p);
+ buf.putLong(lastLink);
+
+ int rowOff = rowSize - written - payloadSize;
+
+ writeFragmentData(row, buf, rowOff, payloadSize);
+ }
+ else {
+ PageUtils.putShort(pageAddr, dataOff, (short)(payloadSize | FRAGMENTED_FLAG));
+
+ PageUtils.putLong(pageAddr, dataOff + 2, lastLink);
+
+ PageUtils.putBytes(pageAddr, dataOff + 10, payload);
+ }
+
+ int itemId = addItem(pageAddr, fullEntrySize, directCnt, indirectCnt, dataOff, pageSize);
+
+ if (row != null)
+ setLink(row, pageAddr, itemId);
+
+ return payloadSize;
+ }
+
+ /**
+ * @param row Row to set link to.
+ * @param pageAddr Page address.
+ * @param itemId Item ID.
+ */
+ private void setLink(CacheDataRow row, long pageAddr, int itemId) {
+ row.link(PageIdUtils.link(getPageId(pageAddr), itemId));
+ }
+
+ /**
+ * Write row data fragment.
+ *
+ * @param row Row.
+ * @param buf Byte buffer.
+ * @param rowOff Offset in row data bytes.
+ * @param payloadSize Data length that should be written in a fragment.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void writeFragmentData(
+ final CacheDataRow row,
+ final ByteBuffer buf,
+ final int rowOff,
+ final int payloadSize
+ ) throws IgniteCheckedException {
+ final int keySize = row.key().valueBytesLength(null);
+ final int valSize = row.value().valueBytesLength(null);
+
+ int written = writeFragment(row, buf, rowOff, payloadSize, EntryPart.CACHE_ID, keySize, valSize);
+ written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.KEY, keySize, valSize);
+ written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.EXPIRE_TIME, keySize, valSize);
+ written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.VALUE, keySize, valSize);
+ written += writeFragment(row, buf, rowOff + written, payloadSize - written, EntryPart.VERSION, keySize, valSize);
+
+ assert written == payloadSize;
+ }
+
+ /**
+ * Try to write fragment data.
+ *
+ * @param rowOff Offset in row data bytes.
+ * @param payloadSize Data length that should be written in this fragment.
+ * @param type Type of the part of entry.
+ * @return Actually written data.
+ * @throws IgniteCheckedException If fail.
+ */
+ private int writeFragment(
+ final CacheDataRow row,
+ final ByteBuffer buf,
+ final int rowOff,
+ final int payloadSize,
+ final EntryPart type,
+ final int keySize,
+ final int valSize
+ ) throws IgniteCheckedException {
+ if (payloadSize == 0)
+ return 0;
+
+ final int prevLen;
+ final int curLen;
+
+ int cacheIdSize = row.cacheId() == 0 ? 0 : 4;
+
+ switch (type) {
+ case CACHE_ID:
+ prevLen = 0;
+ curLen = cacheIdSize;
+
+ break;
+
+ case KEY:
+ prevLen = cacheIdSize;
+ curLen = cacheIdSize + keySize;
+
+ break;
+
+ case EXPIRE_TIME:
+ prevLen = cacheIdSize + keySize;
+ curLen = cacheIdSize + keySize + 8;
+
+ break;
+
+ case VALUE:
+ prevLen = cacheIdSize + keySize + 8;
+ curLen = cacheIdSize + keySize + valSize + 8;
+
+ break;
+
+ case VERSION:
+ prevLen = cacheIdSize + keySize + valSize + 8;
+ curLen = cacheIdSize + keySize + valSize + CacheVersionIO.size(row.version(), false) + 8;
+
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown entry part type: " + type);
+ }
+
+ if (curLen <= rowOff)
+ return 0;
+
+ final int len = Math.min(curLen - rowOff, payloadSize);
+
+ if (type == EntryPart.EXPIRE_TIME)
+ writeExpireTimeFragment(buf, row.expireTime(), rowOff, len, prevLen);
+ else if (type == EntryPart.CACHE_ID)
+ writeCacheIdFragment(buf, row.cacheId(), rowOff, len, prevLen);
+ else if (type != EntryPart.VERSION) {
+ // Write key or value.
+ final CacheObject co = type == EntryPart.KEY ? row.key() : row.value();
+
+ co.putValue(buf, rowOff - prevLen, len);
+ }
+ else
+ writeVersionFragment(buf, row.version(), rowOff, len, prevLen);
+
+ return len;
+ }
+
+ /**
+ * @param buf Byte buffer.
+ * @param ver Version.
+ * @param rowOff Row offset.
+ * @param len Length.
+ * @param prevLen previous length.
+ */
+ private void writeVersionFragment(ByteBuffer buf, GridCacheVersion ver, int rowOff, int len, int prevLen) {
+ int verSize = CacheVersionIO.size(ver, false);
+
+ assert len <= verSize: len;
+
+ if (verSize == len) { // Here we check for equality but not <= because version is the last.
+ // Here we can write version directly.
+ CacheVersionIO.write(buf, ver, false);
+ }
+ else {
+ // We are in the middle of cache version.
+ ByteBuffer verBuf = ByteBuffer.allocate(verSize);
+
+ verBuf.order(buf.order());
+
+ CacheVersionIO.write(verBuf, ver, false);
+
+ buf.put(verBuf.array(), rowOff - prevLen, len);
+ }
+ }
+
+ /**
+ * @param buf Byte buffer.
+ * @param expireTime Expire time.
+ * @param rowOff Row offset.
+ * @param len Length.
+ * @param prevLen previous length.
+ */
+ private void writeExpireTimeFragment(ByteBuffer buf, long expireTime, int rowOff, int len, int prevLen) {
+ int size = 8;
+
+ if (size <= len)
+ buf.putLong(expireTime);
+ else {
+ ByteBuffer timeBuf = ByteBuffer.allocate(size);
+
+ timeBuf.order(buf.order());
+
+ timeBuf.putLong(expireTime);
+
+ buf.put(timeBuf.array(), rowOff - prevLen, len);
+ }
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param cacheId Cache ID.
+ * @param rowOff Row offset.
+ * @param len Length.
+ * @param prevLen Prev length.
+ */
+ private void writeCacheIdFragment(ByteBuffer buf, int cacheId, int rowOff, int len, int prevLen) {
+ if (cacheId == 0)
+ return;
+
+ int size = 4;
+
+ if (size <= len)
+ buf.putInt(cacheId);
+ else {
+ ByteBuffer cacheIdBuf = ByteBuffer.allocate(size);
+
+ cacheIdBuf.order(buf.order());
+
+ cacheIdBuf.putInt(cacheId);
+
+ buf.put(cacheIdBuf.array(), rowOff - prevLen, len);
+ }
+ }
+
+ /**
+ *
+ */
+ private enum EntryPart {
+ /** */
+ KEY,
+
+ /** */
+ VALUE,
+
+ /** */
+ VERSION,
+
+ /** */
+ EXPIRE_TIME,
+
+ /** */
+ CACHE_ID
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param directCnt Direct items count.
+ * @param indirectCnt Indirect items count.
+ * @param pageSize Page size.
+ * @return Item ID (insertion index).
+ */
+ private int insertItem(long pageAddr, int dataOff, int directCnt, int indirectCnt, int pageSize) {
+ if (indirectCnt > 0) {
+ // If the first indirect item is on correct place to become the last direct item, do the transition
+ // and insert the new item into the free slot which was referenced by this first indirect item.
+ short item = getItem(pageAddr, directCnt);
+
+ if (itemId(item) == directCnt) {
+ int directItemIdx = directItemIndex(item);
+
+ setItem(pageAddr, directCnt, getItem(pageAddr, directItemIdx));
+ setItem(pageAddr, directItemIdx, directItemFromOffset(dataOff));
+
+ setDirectCount(pageAddr, directCnt + 1);
+ setIndirectCount(pageAddr, indirectCnt - 1);
+
+ return directItemIdx;
+ }
+ }
+
+ // Move all the indirect items forward to make a free slot and insert new item at the end of direct items.
+ moveItems(pageAddr, directCnt, indirectCnt, +1, pageSize);
+
+ setItem(pageAddr, directCnt, directItemFromOffset(dataOff));
+
+ setDirectCount(pageAddr, directCnt + 1);
+ assert getDirectCount(pageAddr) == directCnt + 1;
+
+ return directCnt; // Previous directCnt will be our itemId.
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param directCnt Direct items count.
+ * @param pageSize Page size.
+ * @return New first entry offset.
+ */
+ private int compactDataEntries(long pageAddr, int directCnt, int pageSize) {
+ assert checkCount(directCnt): directCnt;
+
+ int[] offs = new int[directCnt];
+
+ for (int i = 0; i < directCnt; i++) {
+ int off = directItemToOffset(getItem(pageAddr, i));
+
+ offs[i] = (off << 8) | i; // This way we'll be able to sort by offset using Arrays.sort(...).
+ }
+
+ Arrays.sort(offs);
+
+ // Move right all of the entries if possible to make the page as compact as possible to its tail.
+ int prevOff = pageSize;
+
+ final int start = directCnt - 1;
+ int curOff = offs[start] >>> 8;
+ int curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ for (int i = start; i >= 0; i--) {
+ assert curOff < prevOff : curOff;
+
+ int delta = prevOff - (curOff + curEntrySize);
+
+ int off = curOff;
+ int entrySize = curEntrySize;
+
+ if (delta != 0) { // Move right.
+ assert delta > 0: delta;
+
+ int itemId = offs[i] & 0xFF;
+
+ setItem(pageAddr, itemId, directItemFromOffset(curOff + delta));
+
+ for (int j = i - 1; j >= 0; j--) {
+ int offNext = offs[j] >>> 8;
+ int nextSize = getPageEntrySize(pageAddr, offNext, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ if (offNext + nextSize == off) {
+ i--;
+
+ off = offNext;
+ entrySize += nextSize;
+
+ itemId = offs[j] & 0xFF;
+ setItem(pageAddr, itemId, directItemFromOffset(offNext + delta));
+ }
+ else {
+ curOff = offNext;
+ curEntrySize = nextSize;
+
+ break;
+ }
+ }
+
+ moveBytes(pageAddr, off, entrySize, delta, pageSize);
+
+ off += delta;
+ }
+ else if (i > 0) {
+ curOff = offs[i - 1] >>> 8;
+ curEntrySize = getPageEntrySize(pageAddr, curOff, SHOW_PAYLOAD_LEN | SHOW_LINK);
+ }
+
+ prevOff = off;
+ }
+
+ return prevOff;
+ }
+
+ /**
+ * Full-scan free space calculation procedure.
+ *
+ * @param pageAddr Page to scan.
+ * @param pageSize Page size.
+ * @return Actual free space in the buffer.
+ */
+ private int actualFreeSpace(long pageAddr, int pageSize) {
+ int directCnt = getDirectCount(pageAddr);
+
+ int entriesSize = 0;
+
+ for (int i = 0; i < directCnt; i++) {
+ int off = directItemToOffset(getItem(pageAddr, i));
+
+ int entrySize = getPageEntrySize(pageAddr, off, SHOW_PAYLOAD_LEN | SHOW_LINK);
+
+ entriesSize += entrySize;
+ }
+
+ return pageSize - ITEMS_OFF - entriesSize - (directCnt + getIndirectCount(pageAddr)) * ITEM_SIZE;
+ }
+
+ /**
+ * @param addr Address.
+ * @param off Offset.
+ * @param cnt Count.
+ * @param step Step.
+ * @param pageSize Page size.
+ */
+ private void moveBytes(long addr, int off, int cnt, int step, int pageSize) {
+ assert step != 0: step;
+ assert off + step >= 0;
+ assert off + step + cnt <= pageSize : "[off=" + off + ", step=" + step + ", cnt=" + cnt +
+ ", cap=" + pageSize + ']';
+
+ PageHandler.copyMemory(addr, addr, off, off + step, cnt);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param payloadSize Payload size.
+ * @param row Data row.
+ * @param newRow {@code False} if existing cache entry is updated, in this case skip key data write.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void writeRowData(
+ long pageAddr,
+ int dataOff,
+ int payloadSize,
+ CacheDataRow row,
+ boolean newRow
+ ) throws IgniteCheckedException {
+ long addr = pageAddr + dataOff;
+
+ int cacheIdSize = row.cacheId() != 0 ? 4 : 0;
+
+ if (newRow) {
+ PageUtils.putShort(addr, 0, (short)payloadSize);
+ addr += 2;
+
+ if (cacheIdSize != 0) {
+ PageUtils.putInt(addr, 0, row.cacheId());
+
+ addr += cacheIdSize;
+ }
+
+ addr += row.key().putValue(addr);
+ }
+ else
+ addr += (2 + cacheIdSize + row.key().valueBytesLength(null));
+
+ addr += row.value().putValue(addr);
+
+ CacheVersionIO.write(addr, row.version(), false);
+ addr += CacheVersionIO.size(row.version(), false);
+
+ PageUtils.putLong(addr, 0, row.expireTime());
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param dataOff Data offset.
+ * @param payload Payload
+ */
+ private void writeRowData(
+ long pageAddr,
+ int dataOff,
+ byte[] payload
+ ) {
+ PageUtils.putShort(pageAddr, dataOff, (short)payload.length);
+ dataOff += 2;
+
+ PageUtils.putBytes(pageAddr, dataOff, payload);
+ }
+
+ /**
+ * Defines closure interface for applying computations to data page items.
+ *
+ * @param <T> Closure return type.
+ */
+ public interface CC<T> {
+ /**
+ * Closure body.
+ *
+ * @param link Link to item.
+ * @return Closure return value.
+ * @throws IgniteCheckedException In case of error in closure body.
+ */
+ public T apply(long link) throws IgniteCheckedException;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPagePayload.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPagePayload.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPagePayload.java
new file mode 100644
index 0000000..49eed88
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPagePayload.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+/**
+ *
+ */
+public class DataPagePayload {
+ /** */
+ private final int off;
+
+ /** */
+ private final int payloadSize;
+
+ /** */
+ private final long nextLink;
+
+ /**
+ * @param off Offset.
+ * @param payloadSize Payload size.
+ * @param nextLink Next link.
+ */
+ DataPagePayload(int off, int payloadSize, long nextLink) {
+ this.off = off;
+ this.payloadSize = payloadSize;
+ this.nextLink = nextLink;
+ }
+
+ /**
+ * @return Offset.
+ */
+ public int offset() {
+ return off;
+ }
+
+ /**
+ * @return Payload size.
+ */
+ public int payloadSize() {
+ return payloadSize;
+ }
+
+ /**
+ * @return Link to the next fragment or {@code 0} if it is the last fragment or the data row is not fragmented.
+ */
+ public long nextLink() {
+ return nextLink;
+ }
+}
[19/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
new file mode 100644
index 0000000..4d5eba7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteChangeGlobalStateTest.java
@@ -0,0 +1,711 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteAtomicSequence;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteServices;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.util.typedef.F;
+
+import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
+import static org.apache.ignite.testframework.GridTestUtils.runAsync;
+
+/**
+ *
+ */
+public class IgniteChangeGlobalStateTest extends IgniteChangeGlobalStateAbstractTest {
+ /**
+ * @throws Exception if fail.
+ */
+ public void testStopPrimaryAndActivateFromServerNode() throws Exception {
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ assertTrue(ig1P.active());
+ assertTrue(ig2P.active());
+ assertTrue(ig3P.active());
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ stopAllPrimary();
+
+ ig2B.active(true);
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testStopPrimaryAndActivateFromClientNode() throws Exception {
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1C = backUpClient(0);
+ Ignite ig2C = backUpClient(1);
+ Ignite ig3C = backUpClient(2);
+
+ assertTrue(ig1P.active());
+ assertTrue(ig2P.active());
+ assertTrue(ig3P.active());
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ stopAllPrimary();
+
+ ig2B.active(true);
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+
+ assertTrue(ig1C.active());
+ assertTrue(ig2C.active());
+ assertTrue(ig3C.active());
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testConcurrentActivateFromClientNodeAndServerNode() throws Exception {
+ final Ignite ig1B = backUp(0);
+ final Ignite ig2B = backUp(1);
+ final Ignite ig3B = backUp(2);
+
+ final Ignite ig1C = backUpClient(0);
+ final Ignite ig2C = backUpClient(1);
+ final Ignite ig3C = backUpClient(2);
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ stopAllPrimary();
+
+ final AtomicInteger exCnt = new AtomicInteger();
+
+ final CyclicBarrier barrier = new CyclicBarrier(backUpNodes() + backUpClientNodes());
+
+ IgniteInternalFuture<Void> f1 = runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try {
+ barrier.await();
+ ig1B.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ return null;
+ }
+ });
+
+ IgniteInternalFuture<Void> f2 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig2B.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ IgniteInternalFuture<Void> f3 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig3B.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ IgniteInternalFuture<Void> f4 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig1C.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ IgniteInternalFuture<Void> f5 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig2C.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ IgniteInternalFuture<Void> f6 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig3C.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ f1.get();
+ f2.get();
+ f3.get();
+
+ f4.get();
+ f5.get();
+ f6.get();
+
+ assertTrue(exCnt.get() > 0);
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+
+ assertTrue(ig1C.active());
+ assertTrue(ig2C.active());
+ assertTrue(ig3C.active());
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testConcurrentActivateFromServerNode() throws Exception {
+ final Ignite ig1B = backUp(0);
+ final Ignite ig2B = backUp(1);
+ final Ignite ig3B = backUp(2);
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ stopAllPrimary();
+
+ final AtomicInteger exCnt = new AtomicInteger();
+
+ final CyclicBarrier barrier = new CyclicBarrier(3);
+
+ IgniteInternalFuture<Void> act1 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig1B.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ IgniteInternalFuture<Void> act2 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig2B.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ IgniteInternalFuture<Void> act3 = runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ barrier.await();
+ ig3B.active(true);
+ }catch (Exception e){
+ exCnt.incrementAndGet();
+ }
+ }
+ });
+
+ act1.get();
+ act2.get();
+ act3.get();
+
+ assertEquals(2, exCnt.get());
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+ }
+
+ /**
+ *
+ */
+ public void testActiveAndInActiveAtTheSameTimeCluster() throws Exception {
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ assertTrue(ig1P.cluster().nodes().size() == 6);
+ assertTrue(ig2P.cluster().nodes().size() == 6);
+ assertTrue(ig3P.cluster().nodes().size() == 6);
+
+ List<ClusterNode> primaryNodes = Arrays.asList(
+ ig1P.cluster().localNode(), ig2P.cluster().localNode(), ig3P.cluster().localNode()
+ );
+
+ List<ClusterNode> backUpNodes = Arrays.asList(
+ ig1B.cluster().localNode(), ig3B.cluster().localNode(), ig3B.cluster().localNode()
+ );
+
+ assertTrue(!ig1P.cluster().forRemotes().nodes().containsAll(backUpNodes));
+ assertTrue(!ig2P.cluster().forRemotes().nodes().containsAll(backUpNodes));
+ assertTrue(!ig3P.cluster().forRemotes().nodes().containsAll(backUpNodes));
+
+ assertTrue(ig1B.cluster().nodes().size() == 6);
+ assertTrue(ig2B.cluster().nodes().size() == 6);
+ assertTrue(ig3B.cluster().nodes().size() == 6);
+
+ assertTrue(!ig1B.cluster().forRemotes().nodes().containsAll(primaryNodes));
+ assertTrue(!ig2B.cluster().forRemotes().nodes().containsAll(primaryNodes));
+ assertTrue(!ig3B.cluster().forRemotes().nodes().containsAll(primaryNodes));
+ }
+
+ /**
+ *
+ */
+ public void testActivateOnAlreadyActivatedCluster() throws Exception {
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1C = backUpClient(0);
+ Ignite ig2C = backUpClient(1);
+ Ignite ig3C = backUpClient(2);
+
+ assertTrue(ig1P.active());
+ assertTrue(ig2P.active());
+ assertTrue(ig3P.active());
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ stopAllPrimary();
+
+ ig2B.active(true);
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+
+ assertTrue(ig1C.active());
+ assertTrue(ig2C.active());
+ assertTrue(ig3C.active());
+
+ ig1B.active(true);
+ ig2B.active(true);
+ ig3B.active(true);
+
+ ig1C.active(true);
+ ig2C.active(true);
+ ig3C.active(true);
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+
+ assertTrue(ig1C.active());
+ assertTrue(ig2C.active());
+ assertTrue(ig3C.active());
+ }
+
+ /**
+ *
+ */
+ public void testTryUseCacheInActiveCluster() throws Exception {
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1C = backUpClient(0);
+ Ignite ig2C = backUpClient(1);
+ Ignite ig3C = backUpClient(2);
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ checkExceptionTryUseCache(ig1B);
+ checkExceptionTryUseCache(ig2B);
+ checkExceptionTryUseCache(ig3B);
+
+ checkExceptionTryUseCache(ig1C);
+ checkExceptionTryUseCache(ig2C);
+ checkExceptionTryUseCache(ig3C);
+ }
+
+ /**
+ * @param ig Ig.
+ */
+ private void checkExceptionTryUseCache(final Ignite ig) {
+ assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ IgniteCache cache = ig.cache("cache");
+ return null;
+ }
+ }, IgniteException.class, "can not perform operation, because cluster inactive");
+ }
+
+ /**
+ *
+ */
+ public void testTryUseServiceInActiveCluster() throws Exception {
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1C = backUpClient(0);
+ Ignite ig2C = backUpClient(1);
+ Ignite ig3C = backUpClient(2);
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ checkExceptionTryUseService(ig1B);
+ checkExceptionTryUseService(ig2B);
+ checkExceptionTryUseService(ig3B);
+
+ checkExceptionTryUseService(ig1C);
+ checkExceptionTryUseService(ig2C);
+ checkExceptionTryUseService(ig3C);
+ }
+
+ /**
+ *
+ */
+ private void checkExceptionTryUseService(final Ignite ig) {
+ assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ IgniteServices srvs = ig.services();
+
+ return null;
+ }
+ }, IgniteException.class, "can not perform operation, because cluster inactive");
+ }
+
+ /**
+ *
+ */
+ public void testTryUseDataStructureInActiveCluster() throws Exception {
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1C = backUpClient(0);
+ Ignite ig2C = backUpClient(1);
+ Ignite ig3C = backUpClient(2);
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ checkExceptionTryUseDataStructure(ig1B);
+ checkExceptionTryUseDataStructure(ig2B);
+ checkExceptionTryUseDataStructure(ig3B);
+
+ checkExceptionTryUseDataStructure(ig1C);
+ checkExceptionTryUseDataStructure(ig2C);
+ checkExceptionTryUseDataStructure(ig3C);
+ }
+
+ /**
+ *
+ */
+ private void checkExceptionTryUseDataStructure(final Ignite ig){
+ assertThrows(log, new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ IgniteAtomicSequence seq = ig.atomicSequence("seq", 0, true);
+ return null;
+ }
+ }, IgniteException.class, "can not perform operation, because cluster inactive");
+ }
+
+ /**
+ *
+ */
+ public void testFailGetLock() throws Exception {
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1C = backUpClient(0);
+ Ignite ig2C = backUpClient(1);
+
+ final Ignite ig3C = backUpClient(2);
+
+ assertTrue(ig1P.active());
+ assertTrue(ig2P.active());
+ assertTrue(ig3P.active());
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+
+ stopPrimary(0);
+
+ boolean exc = false;
+
+ try {
+ ig3C.active(true);
+ }
+ catch (IgniteException e) {
+ exc = true;
+
+ log.error("stack trace from remote node", e);
+
+ for (Throwable t : e.getSuppressed())
+ assertTrue(t.getMessage().contains("can't get lock during"));
+ }
+
+ if (!exc)
+ fail();
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+ }
+
+ /**
+ *
+ */
+ public void testActivateAfterFailGetLock() throws Exception {
+ Ignite ig1P = primary(0);
+ Ignite ig2P = primary(1);
+ Ignite ig3P = primary(2);
+
+ Ignite ig1CP = primaryClient(0);
+ Ignite ig2CP = primaryClient(1);
+ Ignite ig3CP = primaryClient(2);
+
+ Ignite ig1B = backUp(0);
+ Ignite ig2B = backUp(1);
+ Ignite ig3B = backUp(2);
+
+ Ignite ig1CB = backUpClient(0);
+ Ignite ig2CB = backUpClient(1);
+ Ignite ig3CB = backUpClient(2);
+
+ assertTrue(ig1P.active());
+ assertTrue(ig2P.active());
+ assertTrue(ig3P.active());
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1CB.active());
+ assertTrue(!ig2CB.active());
+ assertTrue(!ig3CB.active());
+
+ stopPrimary(0);
+
+ boolean exc = false;
+
+ try {
+ ig3CB.active(true);
+ }
+ catch (IgniteException e) {
+ exc = true;
+
+ log.error("stack trace from remote node", e);
+
+ for (Throwable t : e.getSuppressed())
+ assertTrue(t.getMessage().contains("can't get lock during"));
+ }
+
+ if (!exc)
+ fail();
+
+ assertTrue(!ig1B.active());
+ assertTrue(!ig2B.active());
+ assertTrue(!ig3B.active());
+
+ assertTrue(!ig1CB.active());
+ assertTrue(!ig2CB.active());
+ assertTrue(!ig3CB.active());
+
+ assertTrue(ig2P.active());
+ assertTrue(ig3P.active());
+
+ assertTrue(ig1CP.active());
+ assertTrue(ig2CP.active());
+ assertTrue(ig3CP.active());
+
+ stopAllPrimary();
+
+ ig2CB.active(true);
+
+ assertTrue(ig1B.active());
+ assertTrue(ig2B.active());
+ assertTrue(ig3B.active());
+
+ assertTrue(ig1CB.active());
+ assertTrue(ig2CB.active());
+ assertTrue(ig3CB.active());
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testDeActivateFromServerNode() throws Exception {
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig1.active(false);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testDeActivateFromClientNode() throws Exception {
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ Ignite ig1C = primaryClient(0);
+ Ignite ig2C = primaryClient(1);
+ Ignite ig3C = primaryClient(2);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig1C.active(false);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ assertTrue(!ig1C.active());
+ assertTrue(!ig2C.active());
+ assertTrue(!ig3C.active());
+ }
+
+ /**
+ * @throws Exception if fail.
+ */
+ public void testDeActivateCheckCacheDestroy() throws Exception {
+ String chName = "myCache";
+
+ Ignite ig1 = primary(0);
+ Ignite ig2 = primary(1);
+ Ignite ig3 = primary(2);
+
+ ig1.getOrCreateCache(chName);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig1.active(false);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ IgniteEx ex = (IgniteEx)ig1;
+
+ GridCacheProcessor cache = ex.context().cache();
+
+ assertTrue(F.isEmpty(cache.jcaches()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java
new file mode 100644
index 0000000..2d704db
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/IgniteStandByClusterTest.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster;
+
+import java.util.Arrays;
+import java.util.Map;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class IgniteStandByClusterTest extends GridCommonAbstractTest {
+ private static final TcpDiscoveryIpFinder vmIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(vmIpFinder));
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+ cfg.setConsistentId(igniteInstanceName);
+
+ return cfg;
+ }
+
+ public void testNotStartDynamicCachesOnClientAfterActivation() throws Exception {
+ final String cacheName0 = "cache0";
+ final String cacheName = "cache";
+
+ IgniteConfiguration cfg1 = getConfiguration("serv1");
+ IgniteConfiguration cfg2 = getConfiguration("serv2");
+
+ IgniteConfiguration cfg3 = getConfiguration("client");
+ cfg3.setCacheConfiguration(new CacheConfiguration(cacheName0));
+
+ cfg3.setClientMode(true);
+
+ IgniteEx ig1 = startGrid(cfg1);
+ IgniteEx ig2 = startGrid(cfg2);
+ IgniteEx ig3 = startGrid(cfg3);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ ig3.createCache(new CacheConfiguration<>(cacheName));
+
+ assertNotNull(ig3.cache(cacheName));
+ assertNotNull(ig1.cache(cacheName));
+ assertNotNull(ig2.cache(cacheName));
+
+ assertNotNull(ig1.cache(cacheName0));
+ assertNotNull(ig3.cache(cacheName0));
+ assertNotNull(ig2.cache(cacheName0));
+
+ ig3.active(false);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ assertNotNull(ig1.cache(cacheName));
+ assertNotNull(ig2.cache(cacheName));
+
+ Map<String, GridCacheAdapter<?, ?>> caches = U.field(ig3.context().cache(), "caches");
+
+ // Only system caches and cache0
+ assertTrue(caches.size() == 3);
+
+ assertNull(caches.get(cacheName));
+
+ assertNotNull(caches.get(cacheName0));
+
+ assertNotNull(ig3.cache(cacheName));
+ }
+
+ public void testStaticCacheStartAfterActivationWithCacheFilter() throws Exception {
+ String cache1 = "cache1";
+ String cache2 = "cache2";
+ String cache3 = "cache3";
+
+ IgniteConfiguration cfg1 = getConfiguration("node1");
+
+ cfg1.setCacheConfiguration(
+ new CacheConfiguration(cache1).setNodeFilter(new NodeFilterIgnoreByName("node2")));
+
+ IgniteConfiguration cfg2 = getConfiguration("node2");
+
+ cfg2.setCacheConfiguration(
+ new CacheConfiguration(cache2).setNodeFilter(new NodeFilterIgnoreByName("node3")));
+
+ IgniteConfiguration cfg3 = getConfiguration("node3");
+
+ cfg3.setCacheConfiguration(
+ new CacheConfiguration(cache3).setNodeFilter(new NodeFilterIgnoreByName("node1")));
+
+ IgniteEx ig1 = startGrid(cfg1);
+ IgniteEx ig2 = startGrid(cfg2);
+ IgniteEx ig3 = startGrid(cfg3);
+
+ assertTrue(!ig1.active());
+ assertTrue(!ig2.active());
+ assertTrue(!ig3.active());
+
+ for (IgniteEx ig: Arrays.asList(ig1,ig2,ig3)){
+ Map<String, DynamicCacheDescriptor> desc = U.field(U.field(ig.context().cache(), "cachesInfo"), "registeredCaches");
+
+ assertEquals(0, desc.size());
+ }
+
+ ig3.active(true);
+
+ assertTrue(ig1.active());
+ assertTrue(ig2.active());
+ assertTrue(ig3.active());
+
+ for (IgniteEx ig: Arrays.asList(ig1,ig2,ig3)){
+ Map<String, DynamicCacheDescriptor> desc = U.field(
+ U.field(ig.context().cache(), "cachesInfo"), "registeredCaches");
+
+ assertEquals(5, desc.size());
+
+ Map<String, GridCacheAdapter<?, ?>> caches = U.field(ig.context().cache(), "caches");
+
+ assertEquals(4, caches.keySet().size());
+ }
+
+ Map<String, GridCacheAdapter<?, ?>> caches1 = U.field(ig1.context().cache(), "caches");
+
+ Assert.assertNotNull(caches1.get(cache1));
+ Assert.assertNotNull(caches1.get(cache2));
+ Assert.assertNull(caches1.get(cache3));
+
+ Map<String, GridCacheAdapter<?, ?>> caches2 = U.field(ig2.context().cache(), "caches");
+
+ Assert.assertNull(caches2.get(cache1));
+ Assert.assertNotNull(caches2.get(cache2));
+ Assert.assertNotNull(caches2.get(cache3));
+
+ Map<String, GridCacheAdapter<?, ?>> caches3 = U.field(ig3.context().cache(), "caches");
+
+ Assert.assertNotNull(caches3.get(cache1));
+ Assert.assertNull(caches3.get(cache2));
+ Assert.assertNotNull(caches3.get(cache3));
+ }
+
+ private static class NodeFilterIgnoreByName implements IgnitePredicate<ClusterNode>{
+ private final String name;
+
+ private NodeFilterIgnoreByName(String name) {
+ this.name = name;
+ }
+
+ @Override public boolean apply(ClusterNode node) {
+ return !name.equals(node.consistentId());
+ }
+ }
+
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
+ }
+
+ @Override protected void afterTest() throws Exception {
+ super.beforeTest();
+
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivateExtensionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivateExtensionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivateExtensionTest.java
new file mode 100644
index 0000000..4f4d838
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivateExtensionTest.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.extended;
+
+import java.lang.reflect.Method;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+
+/**
+ *
+ */
+public class GridActivateExtensionTest extends GridCacheAbstractFullApiSelfTest {
+ /** Condition id. */
+ private int condId;
+
+ /** Primary ip finder. */
+ private TcpDiscoveryIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Back up ip finder. */
+ private TcpDiscoveryIpFinder backUpIpFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Back up cluster. */
+ private static Map<String, String> backUpCluster = new LinkedHashMap<>();
+
+ /** Test name. */
+ private final String testName = testName();
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setConsistentId("ConsId" + (condId++));
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder);
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ pCfg.setPersistentStorePath(testName + "/db");
+ pCfg.setWalArchivePath(testName + "/db/wal/archive");
+ pCfg.setWalStorePath(testName + "/db/wal");
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ final MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setInitialSize(5 * 1024 * 1024);
+ memPlcCfg.setMaxSize(5 * 1024 * 1024);
+
+ memPlcCfg.setName("dfltMemPlc");
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+
+ memCfg.setDefaultMemoryPolicyName(memPlcCfg.getName());
+
+ memCfg.setPageSize(1024);
+ memCfg.setConcurrencyLevel(64);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName, true));
+
+ super.beforeTestsStarted();
+
+ if (onlyClient())
+ return;
+
+ int nodes = gridCount();
+
+ if (nodes != condId)
+ fail("checking fail");
+
+ condId = 0;
+
+ log.info("start backup cluster");
+
+ for (String name: backUpCluster.keySet()){
+ String n = name + "backUp";
+
+ IgniteConfiguration cfg = getConfiguration(n);
+ cfg.setActiveOnStart(false);
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
+
+ startGrid(n, cfg);
+
+ backUpCluster.put(name, n);
+ }
+
+ log.info("shutdown main cluster");
+
+ for (String name : backUpCluster.keySet()){
+ log.info(name + " stopping");
+
+ stopGrid(name);
+ }
+
+ Ignite ig = grid(0);
+
+ ig.active(true);
+
+ log.info("activate backUp cluster");
+
+ assertEquals(true, ig.active());
+
+ Class c = getClass();
+
+ if (isNearTest(c)) {
+ for (int i = 0; i < gridCount(); i++) {
+ if (ignite(i).configuration().isClientMode()) {
+ Method m = getClientHasNearCacheMethod();
+
+ Boolean r = (Boolean)m.invoke(this);
+
+ if (r != null && r)
+ ignite(i).createNearCache(null, new NearCacheConfiguration<>());
+ else
+ ignite(i).cache(null);
+
+ break;
+ }
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+
+ backUpCluster.clear();
+
+ condId = 0;
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName, true));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Ignite startGrid(String gridName, GridSpringResourceContext ctx) throws Exception {
+ if (!gridName.contains("backUp"))
+ backUpCluster.put(gridName, gridName);
+
+ return super.startGrid(gridName, ctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteEx grid(String name) {
+ String n = name;
+
+ if (backUpCluster.containsKey(name))
+ n = backUpCluster.get(name);
+
+ return super.grid(n);
+ }
+
+ /**
+ *
+ */
+ public boolean onlyClient(){
+ for (int i = 0; i < gridCount(); i++)
+ if (!grid(i).configuration().isClientMode())
+ return false;
+
+ return true;
+ }
+
+ /**
+ * @param c class.
+ */
+ private boolean isNearTest(Class c) {
+ Class p = c.getSuperclass();
+
+ while (!p.equals(Object.class)) {
+ if (p.equals(GridCacheNearOnlyMultiNodeFullApiSelfTest.class))
+ return true;
+
+ p = p.getSuperclass();
+ }
+
+ return false;
+ }
+
+ /**
+ *
+ */
+ public Method getClientHasNearCacheMethod() throws NoSuchMethodException {
+ Class p = getClass().getSuperclass();
+
+ while (!p.equals(Object.class)) {
+ if (p.equals(GridCacheNearOnlyMultiNodeFullApiSelfTest.class))
+ return p.getDeclaredMethod("clientHasNearCache");
+
+ p = p.getSuperclass();
+ }
+
+ return null;
+ }
+
+ /**
+ *
+ */
+ protected String testName() {
+ return getClass().getSimpleName();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationAtomicCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationAtomicCacheSuit.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationAtomicCacheSuit.java
new file mode 100644
index 0000000..99b1152
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationAtomicCacheSuit.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.extended;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicNearEnabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicCopyOnReadDisabledMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearEnabledMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearOnlyMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearOnlyMultiNodeP2PDisabledFullApiSelfTest;
+
+/**
+ *
+ */
+public class GridActivationAtomicCacheSuit extends GridActivationCacheAbstractTestSuit {
+ static {
+ addTest(GridCacheAtomicClientOnlyMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheAtomicClientOnlyMultiNodeP2PDisabledFullApiSelfTest.class);
+ addTest(GridCacheAtomicCopyOnReadDisabledMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheAtomicFullApiSelfTest.class);
+ addTest(GridCacheAtomicMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheAtomicMultiNodeP2PDisabledFullApiSelfTest.class);
+ addTest(GridCacheAtomicNearEnabledFullApiSelfTest.class);
+ addTest(GridCacheAtomicNearEnabledMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheAtomicNearOnlyMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheAtomicNearOnlyMultiNodeP2PDisabledFullApiSelfTest.class);
+
+// addTest(GridCacheAtomicClientOnlyFairAffinityMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicClientOnlyMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicClientOnlyMultiJvmP2PDisabledFullApiSelfTest.class));
+// addTest(GridCacheAtomicCopyOnReadDisabledMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicFairAffinityMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicMultiJvmP2PDisabledFullApiSelfTest.class));
+// addTest(GridCacheAtomicOffHeapMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicOffHeapTieredMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicPrimaryWriteOrderFairAffinityMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicPrimaryWriteOrderMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicPrimaryWriteOrderMultiJvmP2PDisabledFullApiSelfTest.class));
+// addTest(GridCacheAtomicPrimaryWrityOrderOffHeapMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicPrimaryWrityOrderOffHeapTieredMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicNearEnabledFairAffinityMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicNearEnabledMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicNearEnabledPrimaryWriteOrderMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicNearOnlyMultiJvmFullApiSelfTest.class));
+// addTest(GridCacheAtomicNearOnlyMultiJvmP2PDisabledFullApiSelfTest.class));
+ }
+
+ /**
+ *
+ */
+ public static TestSuite suite() {
+ TestSuite suite = buildSuite();
+
+ suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Atomic Cache");
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationCacheAbstractTestSuit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationCacheAbstractTestSuit.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationCacheAbstractTestSuit.java
new file mode 100644
index 0000000..9380745
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationCacheAbstractTestSuit.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.extended;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import javassist.CannotCompileException;
+import javassist.ClassClassPath;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.NotFoundException;
+import junit.framework.TestSuite;
+import org.apache.ignite.IgniteException;
+
+/**
+ *
+ */
+public abstract class GridActivationCacheAbstractTestSuit {
+ /** Tests. */
+ private static final List<Class> tests = new ArrayList<>();
+
+ /** Suffix. */
+ private static final String SUFFIX = "Ex";
+
+ /**
+ * @return Suite.
+ */
+ protected static TestSuite buildSuite() {
+ TestSuite suite = new TestSuite();
+
+ for (Class c : tests)
+ suite.addTestSuite(c);
+
+ return suite;
+ }
+
+ /**
+ * @param c Class.
+ */
+ protected static void addTest(Class c){
+ tests.add(transform(c));
+ }
+
+
+ /**
+ * @param c Class to transform.
+ * @return Transformed class.
+ */
+ private static Class transform(Class c){
+ try {
+ ClassPool pool = ClassPool.getDefault();
+
+ pool.insertClassPath(new ClassClassPath(GridActivationCacheAbstractTestSuit.class));
+
+ String path = c.getProtectionDomain().getCodeSource().getLocation().getPath();
+
+ CtClass ct = pool.get(c.getName());
+
+ String name = c.getName() + SUFFIX;
+
+ CtClass pr = pool.get(GridActivateExtensionTest.class.getName());
+
+ pr.setName(name);
+
+ pr.setSuperclass(ct);
+
+ pr.writeFile(path);
+
+ return Class.forName(name);
+ }
+ catch (IOException e) {
+ System.out.println("Io exception: " + e.getMessage());
+
+ throw new IgniteException(e);
+ }
+ catch (CannotCompileException e) {
+ System.out.println("Cannot compile exception: " + e.getMessage());
+
+ throw new IgniteException(e);
+ }
+ catch (NotFoundException e) {
+ System.out.println("Not found exception: " + e.getMessage());
+
+ throw new IgniteException(e);
+ }
+ catch (ClassNotFoundException e) {
+ System.out.println("Class not found exception: " + e.getMessage());
+
+ throw new IgniteException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java
new file mode 100644
index 0000000..4eb8a38
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.extended;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.local.GridCacheLocalFullApiSelfTest;
+
+/**
+ *
+ */
+public class GridActivationLocalAndNearCacheSuit extends GridActivationCacheAbstractTestSuit {
+ static {
+ addTest(GridCacheLocalAtomicFullApiSelfTest.class);
+ addTest(GridCacheLocalFullApiSelfTest.class);
+
+// addTest(GridCacheNearOnlyFairAffinityMultiJvmFullApiSelfTest.class);
+// addTest(GridCacheNearOnlyMultiJvmFullApiSelfTest.class);
+// addTest(GridCacheNearOnlyMultiJvmP2PDisabledFullApiSelfTest.class);
+ }
+
+ /**
+ *
+ */
+ public static TestSuite suite() {
+ TestSuite suite = buildSuite();
+
+ suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Local and Near Cache");
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java
new file mode 100644
index 0000000..c74aada
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationPartitionedCacheSuit.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.extended;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedMultiNodeLongTxTimeoutFullApiTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedClientOnlyNoPrimaryFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedCopyOnReadDisabledMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest;
+
+/**
+ *
+ */
+public class GridActivationPartitionedCacheSuit extends GridActivationCacheAbstractTestSuit {
+ static {
+ addTest(CachePartitionedMultiNodeLongTxTimeoutFullApiTest.class);
+ addTest(CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.class);
+ addTest(GridCacheNearOnlyMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheNearOnlyMultiNodeP2PDisabledFullApiSelfTest.class);
+ addTest(GridCachePartitionedClientOnlyNoPrimaryFullApiSelfTest.class);
+ addTest(GridCachePartitionedCopyOnReadDisabledMultiNodeFullApiSelfTest.class);
+ addTest(GridCachePartitionedFullApiSelfTest.class);
+ addTest(GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.class);
+ addTest(GridCachePartitionedMultiNodeFullApiSelfTest.class);
+ addTest(GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest.class);
+ addTest(GridCachePartitionedNearDisabledFullApiSelfTest.class);
+ addTest(GridCachePartitionedNearDisabledMultiNodeFullApiSelfTest.class);
+ addTest(GridCachePartitionedNearDisabledMultiNodeP2PDisabledFullApiSelfTest.class);
+ addTest(GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest.class);
+
+// addTest(GridCachePartitionedMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedCopyOnReadDisabledMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedFairAffinityMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedMultiJvmP2PDisabledFullApiSelfTest.class);
+// addTest(GridCachePartitionedNearDisabledAtomicOffHeapTieredMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedNearDisabledFairAffinityMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedNearDisabledMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedNearDisabledMultiJvmP2PDisabledFullApiSelfTest.class);
+// addTest(GridCachePartitionedNearDisabledOffHeapMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedNearDisabledOffHeapTieredMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedOffHeapMultiJvmFullApiSelfTest.class);
+// addTest(GridCachePartitionedOffHeapTieredMultiJvmFullApiSelfTest.class);
+ }
+
+ /**
+ *
+ */
+ public static TestSuite suite() {
+ TestSuite suite = buildSuite();
+
+ suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Partitioned Cache");
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationReplicatedCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationReplicatedCacheSuit.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationReplicatedCacheSuit.java
new file mode 100644
index 0000000..127fe9a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/extended/GridActivationReplicatedCacheSuit.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.extended;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.CacheReplicatedRendezvousAffinityExcludeNeighborsMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.CacheReplicatedRendezvousAffinityMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeP2PDisabledFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedNearOnlyMultiNodeFullApiSelfTest;
+
+/**
+ *
+ */
+public class GridActivationReplicatedCacheSuit extends GridActivationCacheAbstractTestSuit {
+ static {
+ addTest(CacheReplicatedRendezvousAffinityExcludeNeighborsMultiNodeFullApiSelfTest.class);
+ addTest(CacheReplicatedRendezvousAffinityMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheReplicatedAtomicFullApiSelfTest.class);
+ addTest(GridCacheReplicatedFullApiSelfTest.class);
+ addTest(GridCacheReplicatedMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheReplicatedAtomicMultiNodeFullApiSelfTest.class);
+ addTest(GridCacheReplicatedMultiNodeP2PDisabledFullApiSelfTest.class);
+ addTest(GridCacheReplicatedNearOnlyMultiNodeFullApiSelfTest.class);
+
+// tests.add(transform(GridCacheReplicatedAtomicMultiJvmFullApiSelfTest.class));
+// tests.add(transform(GridCacheReplicatedAtomicPrimaryWriteOrderMultiJvmFullApiSelfTest.class));
+// tests.add(transform(GridCacheReplicatedMultiJvmFullApiSelfTest.class));
+// tests.add(transform(GridCacheReplicatedMultiJvmP2PDisabledFullApiSelfTest.class));
+// tests.add(transform(GridCacheReplicatedNearOnlyMultiJvmFullApiSelfTest.class));
+// tests.add(transform(GridCacheReplicatedOffHeapMultiJvmFullApiSelfTest.class));
+// tests.add(transform(GridCacheReplicatedOffHeapTieredMultiJvmFullApiSelfTest.class));
+ }
+
+ /**
+ *
+ */
+ public static TestSuite suite() {
+ TestSuite suite = buildSuite();
+
+ suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Replicated Cache");
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToActiveCluster.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToActiveCluster.java
new file mode 100644
index 0000000..fc9d307
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToActiveCluster.java
@@ -0,0 +1,431 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join;
+
+import java.util.Map;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.AbstractNodeJoinTemplate;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class JoinActiveNodeToActiveCluster extends AbstractNodeJoinTemplate {
+ /**
+ *
+ */
+ @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(true),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheOnlySystem()
+ ).nodeConfiguration(
+ cfg(name(3)).setActiveOnStart(true)
+ ).afterNodeJoin(
+ b.checkCacheOnlySystem()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheOnlySystem()
+ );
+
+ return b;
+ }
+
+ /**
+ *
+ */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(true),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheOnlySystem()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /**
+ *
+ */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheNotEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /**
+ *
+ */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ b.checkCacheNotEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /**
+ *
+ */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(atomicCfg()),
+ cfg(name(1)).setActiveOnStart(true),
+ cfg(name(2)).setActiveOnStart(true)
+ ).afterClusterStarted(
+ new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(3, desc.size());
+
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(3, caches.size());
+ }
+ }
+ }
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(transactionCfg())
+ ).afterNodeJoin(
+ b.checkCacheNotEmpty()
+ ).stateAfterJoin(
+ true
+ ).afterDeActivate(
+ b.checkCacheEmpty()
+ ).setEnd(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ // Server node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinWithOutConfiguration() throws Exception {
+ withOutConfigurationTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
+ staticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testStaticCacheConfigurationInCluster() throws Exception {
+ staticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
+ staticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ staticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ // Client node join.
+
+ /**
+ *
+ */
+ @Override public void testJoinClientWithOutConfiguration() throws Exception {
+ joinClientWithOutConfigurationTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
+ joinClientStaticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
+ joinClientStaticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /**
+ *
+ */
+ @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
+ return withOutConfigurationTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
+ return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
+ return staticCacheConfigurationInClusterTemplate()
+ .nodeConfiguration(setClient)
+ .afterActivate(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient()) {
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+ else {
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+ }
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ }
+ }).afterNodeJoin(
+ new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient()) {
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+ else {
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+ }
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ }
+ }).setEnd(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient()) {
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+ else {
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNull(ig.context().cache().cache(cache2));
+ }
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ }
+ });
+ }
+
+ @Override
+ public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return staticCacheConfigurationDifferentOnBothTemplate()
+ .nodeConfiguration(setClient)
+ .afterActivate(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 3; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient()) {
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+ else {
+ Assert.assertNull(ig.context().cache().cache(cache1));
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+ }
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ Assert.assertEquals(4, caches.size());
+ }
+ }
+ })
+ .afterNodeJoin(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 4; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertEquals(4, caches.size());
+ else
+ Assert.assertEquals(3, caches.size());
+ }
+ }
+ })
+ .setEnd(new Runnable() {
+ @Override public void run() {
+ for (int i = 0; i < 4; i++) {
+ IgniteEx ig = grid(name(i));
+
+ Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
+
+ Assert.assertEquals(4, desc.size());
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertNotNull(ig.context().cache().cache(cache1));
+
+ Assert.assertNotNull(ig.context().cache().cache(cache2));
+
+ Map<String, GridCacheAdapter> caches = caches(ig);
+
+ if (!ig.context().discovery().localNode().isClient())
+ Assert.assertEquals(4, caches.size());
+ else
+ Assert.assertEquals(3, caches.size());
+ }
+ }
+ });
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
+ return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToInActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToInActiveCluster.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToInActiveCluster.java
new file mode 100644
index 0000000..b248b21
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/standbycluster/join/JoinActiveNodeToInActiveCluster.java
@@ -0,0 +1,227 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.standbycluster.join;
+
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.AbstractNodeJoinTemplate;
+
+/**
+ *
+ */
+public class JoinActiveNodeToInActiveCluster extends AbstractNodeJoinTemplate {
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(false),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheOnlySystem()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0)).setActiveOnStart(false),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3)).setActiveOnStart(true)
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(allCacheConfigurations()),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(allCacheConfigurations())
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+ return b;
+ }
+
+ /** {@inheritDoc} */
+ @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ JoinNodeTestPlanBuilder b = builder();
+
+ b.clusterConfiguration(
+ cfg(name(0))
+ .setActiveOnStart(false)
+ .setCacheConfiguration(atomicCfg()),
+ cfg(name(1)).setActiveOnStart(false),
+ cfg(name(2)).setActiveOnStart(false)
+ ).afterClusterStarted(
+ b.checkCacheEmpty()
+ ).nodeConfiguration(
+ cfg(name(3))
+ .setActiveOnStart(true)
+ .setCacheConfiguration(transactionCfg())
+ ).afterNodeJoin(
+ b.checkCacheEmpty()
+ ).stateAfterJoin(
+ false
+ ).afterActivate(
+ b.checkCacheNotEmpty()
+ );
+
+ return b;
+ }
+
+ // Server node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinWithOutConfiguration() throws Exception {
+ withOutConfigurationTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
+ staticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationInCluster() throws Exception {
+ staticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
+ staticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ staticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ // Client node join.
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientWithOutConfiguration() throws Exception {
+ joinClientWithOutConfigurationTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
+ joinClientStaticCacheConfigurationOnJoinTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
+ joinClientStaticCacheConfigurationInClusterTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
+ joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
+ return withOutConfigurationTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
+ return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
+ return staticCacheConfigurationInClusterTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
+ return staticCacheConfigurationDifferentOnBothTemplate().nodeConfiguration(setClient);
+ }
+
+ @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
+ return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
+ }
+
+}
[40/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 708df49..c72f53e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -38,12 +38,11 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
import org.apache.ignite.internal.processors.cache.GridCacheLockTimeoutException;
-import org.apache.ignite.internal.processors.cache.GridCacheMapEntryFactory;
import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedUnlockRequest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
index 404265d..7da3d4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtDetachedCacheEntry.java
@@ -22,7 +22,7 @@ import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.typedef.internal.S;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
index 467b906..1cc6c28 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionSupplier.java
@@ -30,7 +30,7 @@ import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionTopology;
import org.apache.ignite.internal.util.lang.GridCloseableIterator;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
index b17d0b5..646281b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearCacheEntry.java
@@ -30,7 +30,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvcc;
import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
import org.apache.ignite.internal.processors.cache.GridCacheOperation;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
index 384cd41..730c71d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/local/atomic/GridLocalAtomicCache.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -55,7 +54,7 @@ import org.apache.ignite.internal.processors.cache.GridCachePreloaderAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.local.GridLocalCache;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
new file mode 100644
index 0000000..57aeaef
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRow.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+
+/**
+ * Cache data row.
+ */
+public interface CacheDataRow extends CacheSearchRow {
+ /**
+ * @return Cache value.
+ */
+ public CacheObject value();
+
+ /**
+ * @return Cache entry version.
+ */
+ public GridCacheVersion version();
+
+ /**
+ * @return Expire time.
+ */
+ public long expireTime();
+
+ /**
+ * @return Partition for this key.
+ */
+ public int partition();
+
+ /**
+ * @param link Link for this row.
+ */
+ public void link(long link);
+
+ /**
+ * @param key Key.
+ */
+ public void key(KeyCacheObject key);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java
new file mode 100644
index 0000000..4d75475
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheDataRowAdapter.java
@@ -0,0 +1,593 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.IncompleteCacheObject;
+import org.apache.ignite.internal.processors.cache.IncompleteObject;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.pagemem.PageIdUtils.itemId;
+import static org.apache.ignite.internal.pagemem.PageIdUtils.pageId;
+
+/**
+ * Cache data row adapter.
+ */
+public class CacheDataRowAdapter implements CacheDataRow {
+ /** */
+ @GridToStringExclude
+ protected long link;
+
+ /** */
+ @GridToStringInclude
+ protected KeyCacheObject key;
+
+ /** */
+ @GridToStringInclude
+ protected CacheObject val;
+
+ /** */
+ protected long expireTime = -1;
+
+ /** */
+ @GridToStringInclude
+ protected GridCacheVersion ver;
+
+ /** */
+ @GridToStringInclude
+ protected int cacheId;
+
+ /**
+ * @param link Link.
+ */
+ public CacheDataRowAdapter(long link) {
+ // Link can be 0 here.
+ this.link = link;
+ }
+
+ /**
+ * @param key Key.
+ * @param val Value.
+ * @param expireTime Expire time.
+ * @param ver Version.
+ */
+ public CacheDataRowAdapter(KeyCacheObject key, CacheObject val, GridCacheVersion ver, long expireTime) {
+ this.key = key;
+ this.val = val;
+ this.ver = ver;
+ this.expireTime = expireTime;
+ }
+
+ /**
+ * Read row from data pages.
+ *
+ * @param grp Cache group.
+ * @param rowData Required row data.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final void initFromLink(CacheGroupContext grp, RowData rowData) throws IgniteCheckedException {
+ initFromLink(grp, grp.shared(), grp.memoryPolicy().pageMemory(), rowData);
+ }
+
+ /**
+ * Read row from data pages.
+ * Can be called with cctx == null, if cache instance is unknown, but its ID is stored in the data row.
+ *
+ * @param grp Cache group.
+ * @param sharedCtx Shared context.
+ * @param pageMem Page memory.
+ * @param rowData Row data.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final void initFromLink(
+ @Nullable CacheGroupContext grp,
+ GridCacheSharedContext<?, ?> sharedCtx,
+ PageMemory pageMem,
+ RowData rowData)
+ throws IgniteCheckedException {
+ assert link != 0 : "link";
+ assert key == null : "key";
+
+ CacheObjectContext coctx = grp != null ? grp.cacheObjectContext() : null;
+
+ boolean readCacheId = grp == null || grp.storeCacheIdInDataPage();
+
+ long nextLink = link;
+ IncompleteObject<?> incomplete = null;
+ boolean first = true;
+
+ do {
+ final long pageId = pageId(nextLink);
+
+ // Group is null if try evict page, with persistence evictions should be disabled.
+ assert grp != null || !sharedCtx.database().persistenceEnabled();
+
+ int grpId = grp != null ? grp.groupId() : 0;
+
+ final long page = pageMem.acquirePage(grpId, pageId);
+
+ try {
+ long pageAddr = pageMem.readLock(grpId, pageId, page); // Non-empty data page must not be recycled.
+
+ assert pageAddr != 0L : nextLink;
+
+ try {
+ DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
+
+ DataPagePayload data = io.readPayload(pageAddr,
+ itemId(nextLink),
+ pageMem.pageSize());
+
+ nextLink = data.nextLink();
+
+ if (first) {
+ if (nextLink == 0) {
+ // Fast path for a single page row.
+ readFullRow(sharedCtx, coctx, pageAddr + data.offset(), rowData, readCacheId);
+
+ return;
+ }
+
+ first = false;
+ }
+
+ ByteBuffer buf = pageMem.pageBuffer(pageAddr);
+
+ buf.position(data.offset());
+ buf.limit(data.offset() + data.payloadSize());
+
+ boolean keyOnly = rowData == RowData.KEY_ONLY;
+
+ incomplete = readFragment(sharedCtx, coctx, buf, keyOnly, readCacheId, incomplete);
+
+ if (keyOnly && key != null)
+ return;
+ }
+ finally {
+ pageMem.readUnlock(grpId, pageId, page);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, pageId, page);
+ }
+ }
+ while(nextLink != 0);
+
+ assert isReady() : "ready";
+ }
+
+ /**
+ * @param sharedCtx Cache shared context.
+ * @param coctx Cache object context.
+ * @param buf Buffer.
+ * @param keyOnly {@code true} If need to read only key object.
+ * @param readCacheId {@code true} If need to read cache ID.
+ * @param incomplete Incomplete object.
+ * @throws IgniteCheckedException If failed.
+ * @return Read object.
+ */
+ private IncompleteObject<?> readFragment(
+ GridCacheSharedContext<?, ?> sharedCtx,
+ CacheObjectContext coctx,
+ ByteBuffer buf,
+ boolean keyOnly,
+ boolean readCacheId,
+ IncompleteObject<?> incomplete
+ ) throws IgniteCheckedException {
+ if (readCacheId && cacheId == 0) {
+ incomplete = readIncompleteCacheId(buf, incomplete);
+
+ if (cacheId == 0)
+ return incomplete;
+
+ incomplete = null;
+ }
+
+ if (coctx == null) {
+ // coctx can be null only when grp is null too, this means that
+ // we are in process of eviction and cacheId is mandatory part of data.
+ assert cacheId != 0;
+
+ coctx = sharedCtx.cacheContext(cacheId).cacheObjectContext();
+ }
+
+ // Read key.
+ if (key == null) {
+ incomplete = readIncompleteKey(coctx, buf, (IncompleteCacheObject)incomplete);
+
+ if (key == null || keyOnly)
+ return incomplete;
+
+ incomplete = null;
+ }
+
+ if (expireTime == -1) {
+ incomplete = readIncompleteExpireTime(buf, incomplete);
+
+ if (expireTime == -1)
+ return incomplete;
+
+ incomplete = null;
+ }
+
+ // Read value.
+ if (val == null) {
+ incomplete = readIncompleteValue(coctx, buf, (IncompleteCacheObject)incomplete);
+
+ if (val == null)
+ return incomplete;
+
+ incomplete = null;
+ }
+
+ // Read version.
+ if (ver == null)
+ incomplete = readIncompleteVersion(buf, incomplete);
+
+ return incomplete;
+ }
+
+ /**
+ * @param sharedCtx Cache shared context.
+ * @param coctx Cache object context.
+ * @param addr Address.
+ * @param rowData Required row data.
+ * @param readCacheId {@code true} If need to read cache ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void readFullRow(
+ GridCacheSharedContext<?, ?> sharedCtx,
+ CacheObjectContext coctx,
+ long addr,
+ RowData rowData,
+ boolean readCacheId)
+ throws IgniteCheckedException {
+ int off = 0;
+
+ if (readCacheId) {
+ cacheId = PageUtils.getInt(addr, off);
+
+ off += 4;
+ }
+
+ if (coctx == null)
+ coctx = sharedCtx.cacheContext(cacheId).cacheObjectContext();
+
+ int len = PageUtils.getInt(addr, off);
+ off += 4;
+
+ if (rowData != RowData.NO_KEY) {
+ byte type = PageUtils.getByte(addr, off);
+ off++;
+
+ byte[] bytes = PageUtils.getBytes(addr, off, len);
+ off += len;
+
+ key = coctx.kernalContext().cacheObjects().toKeyCacheObject(coctx, type, bytes);
+
+ if (rowData == RowData.KEY_ONLY)
+ return;
+ }
+ else
+ off += len + 1;
+
+ len = PageUtils.getInt(addr, off);
+ off += 4;
+
+ byte type = PageUtils.getByte(addr, off);
+ off++;
+
+ byte[] bytes = PageUtils.getBytes(addr, off, len);
+ off += len;
+
+ val = coctx.kernalContext().cacheObjects().toCacheObject(coctx, type, bytes);
+
+ ver = CacheVersionIO.read(addr + off, false);
+
+ off += CacheVersionIO.size(ver, false);
+
+ expireTime = PageUtils.getLong(addr, off);
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param incomplete Incomplete.
+ */
+ private IncompleteObject<?> readIncompleteCacheId(
+ ByteBuffer buf,
+ IncompleteObject<?> incomplete
+ ) {
+ if (incomplete == null) {
+ int remaining = buf.remaining();
+
+ if (remaining == 0)
+ return null;
+
+ int size = 4;
+
+ if (remaining >= size) {
+ cacheId = buf.getInt();
+
+ assert cacheId != 0;
+
+ return null;
+ }
+
+ incomplete = new IncompleteObject<>(new byte[size]);
+ }
+
+ incomplete.readData(buf);
+
+ if (incomplete.isReady()) {
+ final ByteBuffer timeBuf = ByteBuffer.wrap(incomplete.data());
+
+ timeBuf.order(buf.order());
+
+ cacheId = timeBuf.getInt();
+
+ assert cacheId != 0;
+ }
+
+ return incomplete;
+ }
+
+ /**
+ * @param coctx Cache object context.
+ * @param buf Buffer.
+ * @param incomplete Incomplete object.
+ * @return Incomplete object.
+ * @throws IgniteCheckedException If failed.
+ */
+ private IncompleteCacheObject readIncompleteKey(
+ CacheObjectContext coctx,
+ ByteBuffer buf,
+ IncompleteCacheObject incomplete
+ ) throws IgniteCheckedException {
+ incomplete = coctx.kernalContext().cacheObjects().toKeyCacheObject(coctx, buf, incomplete);
+
+ if (incomplete.isReady()) {
+ key = (KeyCacheObject)incomplete.object();
+
+ assert key != null;
+ }
+ else
+ assert !buf.hasRemaining();
+
+ return incomplete;
+ }
+
+ /**
+ * @param coctx Cache object context.
+ * @param buf Buffer.
+ * @param incomplete Incomplete object.
+ * @return Incomplete object.
+ * @throws IgniteCheckedException If failed.
+ */
+ private IncompleteCacheObject readIncompleteValue(
+ CacheObjectContext coctx,
+ ByteBuffer buf,
+ IncompleteCacheObject incomplete
+ ) throws IgniteCheckedException {
+ incomplete = coctx.kernalContext().cacheObjects().toCacheObject(coctx, buf, incomplete);
+
+ if (incomplete.isReady()) {
+ val = incomplete.object();
+
+ assert val != null;
+ }
+ else
+ assert !buf.hasRemaining();
+
+ return incomplete;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param incomplete Incomplete object.
+ * @return Incomplete object.
+ */
+ private IncompleteObject<?> readIncompleteExpireTime(
+ ByteBuffer buf,
+ IncompleteObject<?> incomplete
+ ) {
+ if (incomplete == null) {
+ int remaining = buf.remaining();
+
+ if (remaining == 0)
+ return null;
+
+ int size = 8;
+
+ if (remaining >= size) {
+ expireTime = buf.getLong();
+
+ assert expireTime >= 0 : expireTime;
+
+ return null;
+ }
+
+ incomplete = new IncompleteObject<>(new byte[size]);
+ }
+
+ incomplete.readData(buf);
+
+ if (incomplete.isReady()) {
+ final ByteBuffer timeBuf = ByteBuffer.wrap(incomplete.data());
+
+ timeBuf.order(buf.order());
+
+ expireTime = timeBuf.getLong();
+
+ assert expireTime >= 0;
+ }
+
+ return incomplete;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param incomplete Incomplete object.
+ * @return Incomplete object.
+ * @throws IgniteCheckedException If failed.
+ */
+ private IncompleteObject<?> readIncompleteVersion(
+ ByteBuffer buf,
+ IncompleteObject<?> incomplete
+ ) throws IgniteCheckedException {
+ if (incomplete == null) {
+ int remaining = buf.remaining();
+
+ if (remaining == 0)
+ return null;
+
+ int size = CacheVersionIO.readSize(buf, false);
+
+ if (remaining >= size) {
+ // If the whole version is on a single page, just read it.
+ ver = CacheVersionIO.read(buf, false);
+
+ assert !buf.hasRemaining(): buf.remaining();
+ assert ver != null;
+
+ return null;
+ }
+
+ // We have to read multipart version.
+ incomplete = new IncompleteObject<>(new byte[size]);
+ }
+
+ incomplete.readData(buf);
+
+ if (incomplete.isReady()) {
+ final ByteBuffer verBuf = ByteBuffer.wrap(incomplete.data());
+
+ verBuf.order(buf.order());
+
+ ver = CacheVersionIO.read(verBuf, false);
+
+ assert ver != null;
+ }
+
+ assert !buf.hasRemaining();
+
+ return incomplete;
+ }
+
+ /**
+ * @return {@code True} if entry is ready.
+ */
+ public boolean isReady() {
+ return ver != null && val != null && key != null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public KeyCacheObject key() {
+ assert key != null : "Key is not ready: " + this;
+
+ return key;
+ }
+
+ /**
+ * @param key Key.
+ */
+ @Override public void key(KeyCacheObject key) {
+ assert key != null;
+
+ this.key = key;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int cacheId() {
+ return cacheId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public CacheObject value() {
+ assert val != null : "Value is not ready: " + this;
+
+ return val;
+ }
+
+ /** {@inheritDoc} */
+ @Override public GridCacheVersion version() {
+ assert ver != null : "Version is not ready: " + this;
+
+ return ver;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long expireTime() {
+ return expireTime;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int partition() {
+ return PageIdUtils.partId(link);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long link() {
+ return link;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void link(long link) {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hash() {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ *
+ */
+ public enum RowData {
+ /** */
+ FULL,
+
+ /** */
+ KEY_ONLY,
+
+ /** */
+ NO_KEY
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(CacheDataRowAdapter.class, this, "link", U.hexLong(link));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheSearchRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheSearchRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheSearchRow.java
new file mode 100644
index 0000000..1637eb0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CacheSearchRow.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+
+/**
+ *
+ */
+public interface CacheSearchRow {
+ /**
+ * @return Cache key.
+ */
+ public KeyCacheObject key();
+
+ /**
+ * @return Link for this row.
+ */
+ public long link();
+
+ /**
+ * @return Key hash code.
+ */
+ public int hash();
+
+ /**
+ * @return Cache ID or {@code 0} if cache ID is not defined.
+ */
+ public int cacheId();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointLockStateChecker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointLockStateChecker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointLockStateChecker.java
new file mode 100644
index 0000000..ca6fb69
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/CheckpointLockStateChecker.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+/**
+ * Interface for perform checking that checkpoint lock is held by current track
+ */
+public interface CheckpointLockStateChecker {
+ /**
+ * @return true if checkpoint lock is held by current thread
+ */
+ public boolean checkpointLockIsHeldByThread();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
new file mode 100644
index 0000000..b2a8f36
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DataStructure.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.RecycleRecord;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.INDEX_PARTITION;
+import static org.apache.ignite.internal.pagemem.PageIdAllocator.MAX_PARTITION_ID;
+
+/**
+ * Base class for all the data structures based on {@link PageMemory}.
+ */
+public abstract class DataStructure implements PageLockListener {
+ /** For tests. */
+ public static Random rnd;
+
+ /** */
+ protected final int cacheId;
+
+ /** */
+ protected final PageMemory pageMem;
+
+ /** */
+ protected final IgniteWriteAheadLogManager wal;
+
+ /** */
+ protected ReuseList reuseList;
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageMem Page memory.
+ * @param wal Write ahead log manager.
+ */
+ public DataStructure(
+ int cacheId,
+ PageMemory pageMem,
+ IgniteWriteAheadLogManager wal
+ ) {
+ assert pageMem != null;
+
+ this.cacheId = cacheId;
+ this.pageMem = pageMem;
+ this.wal = wal;
+ }
+
+ /**
+ * @return Cache ID.
+ */
+ public final int getCacheId() {
+ return cacheId;
+ }
+
+ /**
+ * @param max Max.
+ * @return Random value from {@code 0} (inclusive) to the given max value (exclusive).
+ */
+ public static int randomInt(int max) {
+ Random rnd0 = rnd != null ? rnd : ThreadLocalRandom.current();
+
+ return rnd0.nextInt(max);
+ }
+
+ /**
+ * @param bag Reuse bag.
+ * @return Allocated page.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final long allocatePage(ReuseBag bag) throws IgniteCheckedException {
+ long pageId = bag != null ? bag.pollFreePage() : 0;
+
+ if (pageId == 0 && reuseList != null)
+ pageId = reuseList.takeRecycledPage();
+
+ if (pageId == 0)
+ pageId = allocatePageNoReuse();
+
+ assert pageId != 0;
+
+ return pageId;
+ }
+
+ /**
+ * @return Page ID of newly allocated page.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected long allocatePageNoReuse() throws IgniteCheckedException {
+ return pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION, FLAG_IDX);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @return Page absolute pointer.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final long acquirePage(long pageId) throws IgniteCheckedException {
+ assert PageIdUtils.flag(pageId) == FLAG_IDX && PageIdUtils.partId(pageId) == INDEX_PARTITION ||
+ PageIdUtils.flag(pageId) == FLAG_DATA && PageIdUtils.partId(pageId) <= MAX_PARTITION_ID : U.hexLong(pageId);
+
+ return pageMem.acquirePage(cacheId, pageId);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ */
+ protected final void releasePage(long pageId, long page) {
+ pageMem.releasePage(cacheId, pageId, page);
+ }
+
+ /**
+ * @param pageId Page ID
+ * @param page Page pointer.
+ * @return Page address or {@code 0} if failed to lock due to recycling.
+ */
+ protected final long tryWriteLock(long pageId, long page) {
+ return PageHandler.writeLock(pageMem, cacheId, pageId, page, this, true);
+ }
+
+ /**
+ * @param pageId Page ID
+ * @param page Page pointer.
+ * @return Page address.
+ */
+ protected final long writeLock(long pageId, long page) {
+ return PageHandler.writeLock(pageMem, cacheId, pageId, page, this, false);
+ }
+
+ /**
+ * <p>
+ * Note: Default WAL record policy will be used.
+ * </p>
+ * @param pageId Page ID
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param dirty Dirty flag.
+ */
+ protected final void writeUnlock(long pageId, long page, long pageAddr, boolean dirty) {
+ writeUnlock(pageId, page, pageAddr, null, dirty);
+ }
+
+ /**
+ * @param pageId Page ID
+ * @param page Page pointer.
+ * @return Page address.
+ */
+ protected final long readLock(long pageId, long page) {
+ return PageHandler.readLock(pageMem, cacheId, pageId, page, this);
+ }
+
+ /**
+ * @param pageId Page ID
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ */
+ protected final void readUnlock(long pageId, long page, long pageAddr) {
+ PageHandler.readUnlock(pageMem, cacheId, pageId, page, pageAddr, this);
+ }
+
+ /**
+ * @param pageId Page ID
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param walPlc Full page WAL record policy.
+ * @param dirty Dirty flag.
+ */
+ protected final void writeUnlock(long pageId, long page, long pageAddr, Boolean walPlc, boolean dirty) {
+ PageHandler.writeUnlock(pageMem, cacheId, pageId, page, pageAddr, this, walPlc, dirty);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param walPlc Full page WAL record policy.
+ * @return {@code true} If we need to make a delta WAL record for the change in this page.
+ */
+ protected final boolean needWalDeltaRecord(long pageId, long page, Boolean walPlc) {
+ return PageHandler.isWalDeltaRecordNeeded(pageMem, cacheId, pageId, page, wal, walPlc);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param h Handler.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final <R> R write(
+ long pageId,
+ PageHandler<?, R> h,
+ int intArg,
+ R lockFailed) throws IgniteCheckedException {
+ return PageHandler.writePage(pageMem, cacheId, pageId, this, h, null, null, null, null, intArg, lockFailed);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param h Handler.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final <X, R> R write(
+ long pageId,
+ PageHandler<X, R> h,
+ X arg,
+ int intArg,
+ R lockFailed) throws IgniteCheckedException {
+ return PageHandler.writePage(pageMem, cacheId, pageId, this, h, null, null, null, arg, intArg, lockFailed);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param h Handler.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final <X, R> R write(
+ long pageId,
+ long page,
+ PageHandler<X, R> h,
+ X arg,
+ int intArg,
+ R lockFailed) throws IgniteCheckedException {
+ return PageHandler.writePage(pageMem, cacheId, pageId, page, this, h, null, null, null, arg, intArg, lockFailed);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param h Handler.
+ * @param init IO for new page initialization or {@code null} if it is an existing page.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final <X, R> R write(
+ long pageId,
+ PageHandler<X, R> h,
+ PageIO init,
+ X arg,
+ int intArg,
+ R lockFailed) throws IgniteCheckedException {
+ return PageHandler.writePage(pageMem, cacheId, pageId, this, h, init, wal, null, arg, intArg, lockFailed);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param h Handler.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final <X, R> R read(
+ long pageId,
+ PageHandler<X, R> h,
+ X arg,
+ int intArg,
+ R lockFailed) throws IgniteCheckedException {
+ return PageHandler.readPage(pageMem, cacheId, pageId, this, h, arg, intArg, lockFailed);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param h Handler.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final <X, R> R read(
+ long pageId,
+ long page,
+ PageHandler<X, R> h,
+ X arg,
+ int intArg,
+ R lockFailed) throws IgniteCheckedException {
+ return PageHandler.readPage(pageMem, cacheId, pageId, page, this, h, arg, intArg, lockFailed);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param init IO for new page initialization.
+ * @throws IgniteCheckedException if failed.
+ */
+ protected final void init(long pageId, PageIO init) throws IgniteCheckedException {
+ PageHandler.initPage(pageMem, cacheId, pageId, init, wal, this);
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param walPlc Full page WAL record policy.
+ * @return Rotated page ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final long recyclePage(
+ long pageId,
+ long page,
+ long pageAddr,
+ Boolean walPlc) throws IgniteCheckedException {
+ long rotated = PageIdUtils.rotatePageId(pageId);
+
+ PageIO.setPageId(pageAddr, rotated);
+
+ if (needWalDeltaRecord(pageId, page, walPlc))
+ wal.log(new RecycleRecord(cacheId, pageId, rotated));
+
+ return rotated;
+ }
+
+ /**
+ * @return Page size.
+ */
+ protected final int pageSize() {
+ return pageMem.pageSize();
+ }
+
+ @Override public void onBeforeWriteLock(int cacheId, long pageId, long page) {
+ // No-op.
+ }
+
+ @Override public void onWriteLock(int cacheId, long pageId, long page, long pageAddr) {
+ // No-op.
+ }
+
+ @Override public void onWriteUnlock(int cacheId, long pageId, long page, long pageAddr) {
+ // No-op.
+ }
+
+ @Override public void onBeforeReadLock(int cacheId, long pageId, long page) {
+ // No-op.
+ }
+
+ @Override public void onReadLock(int cacheId, long pageId, long page, long pageAddr) {
+ // No-op.
+ }
+
+ @Override public void onReadUnlock(int cacheId, long pageId, long page, long pageAddr) {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
new file mode 100644
index 0000000..f4da637
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/DbCheckpointListener.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.util.Map;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.typedef.T2;
+
+/**
+ *
+ */
+public interface DbCheckpointListener {
+ public interface Context {
+ public boolean nextSnapshot();
+
+ public Map<T2<Integer, Integer>, T2<Integer, Integer>> partitionStatMap();
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public void onCheckpointBegin(Context context) throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java
new file mode 100644
index 0000000..c056c52
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/FullPageIdIterableComparator.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.Serializable;
+import java.util.Comparator;
+import org.apache.ignite.internal.util.typedef.T2;
+
+/**
+ *
+ */
+public class FullPageIdIterableComparator implements Comparator<T2<Integer, Integer>>, Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ public static final FullPageIdIterableComparator INSTANCE = new FullPageIdIterableComparator();
+
+ /** {@inheritDoc} */
+ @Override public int compare(T2<Integer, Integer> o1, T2<Integer, Integer> o2) {
+ if (o1.get1() < o2.get1())
+ return -1;
+
+ if (o1.get1() > o2.get1())
+ return 1;
+
+ if (o1.get2() < o2.get2())
+ return -1;
+
+ if (o1.get2() > o2.get2())
+ return 1;
+
+ return 0;
+ }
+}
+
[07/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreDataStructuresTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreDataStructuresTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreDataStructuresTest.java
deleted file mode 100644
index 718cf54..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreDataStructuresTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteAtomicLong;
-import org.apache.ignite.IgniteAtomicSequence;
-import org.apache.ignite.IgniteQueue;
-import org.apache.ignite.IgniteSet;
-import org.apache.ignite.configuration.CollectionConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePersistentStoreDataStructuresTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(200 * 1024 * 1024);
- memPlcCfg.setMaxSize(200 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration().setWalMode(WALMode.LOG_ONLY));
-
- cfg.setActiveOnStart(false);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- GridTestUtils.deleteDbFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- GridTestUtils.deleteDbFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- stopAllGrids();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testQueue() throws Exception {
- Ignite ignite = startGrids(4);
-
- ignite.active(true);
-
- IgniteQueue<Object> queue = ignite.queue("testQueue", 100, new CollectionConfiguration());
-
- for (int i = 0; i < 100; i++)
- queue.offer(i);
-
- stopAllGrids();
-
- ignite = startGrids(4);
-
- ignite.active(true);
-
- queue = ignite.queue("testQueue", 0, null);
-
- for (int i = 0; i < 100; i++)
- assertEquals(i, queue.poll());
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testAtomic() throws Exception {
- Ignite ignite = startGrids(4);
-
- ignite.active(true);
-
- IgniteAtomicLong atomicLong = ignite.atomicLong("testLong", 0, true);
-
- for (int i = 0; i < 100; i++)
- atomicLong.incrementAndGet();
-
- stopAllGrids();
-
- ignite = startGrids(4);
-
- ignite.active(true);
-
- atomicLong = ignite.atomicLong("testLong", 0, false);
-
- for (int i = 100; i != 0; )
- assertEquals(i--, atomicLong.getAndDecrement());
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSequence() throws Exception {
- Ignite ignite = startGrids(4);
-
- ignite.active(true);
-
- IgniteAtomicSequence sequence = ignite.atomicSequence("testSequence", 0, true);
-
- int i = 0;
-
- while (i < 1000) {
- sequence.incrementAndGet();
-
- i++;
- }
-
- stopAllGrids();
-
- ignite = startGrids(4);
-
- ignite.active(true);
-
- sequence = ignite.atomicSequence("testSequence", 0, false);
-
- assertTrue(sequence.incrementAndGet() > i);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testSet() throws Exception {
- Ignite ignite = startGrids(4);
-
- ignite.active(true);
-
- IgniteSet<Object> set = ignite.set("testSet", new CollectionConfiguration());
-
- for (int i = 0; i < 100; i++)
- set.add(i);
-
- stopAllGrids();
-
- ignite = startGrids(4);
-
- ignite.active(true);
-
- set = ignite.set("testSet", null);
-
- assertFalse(set.add(99));
-
- for (int i = 0; i < 100; i++)
- assertTrue(set.contains(i));
-
- assertEquals(100, set.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreSchemaLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreSchemaLoadTest.java
deleted file mode 100644
index bed17ac..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreSchemaLoadTest.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.QueryEntity;
-import org.apache.ignite.cache.QueryIndex;
-import org.apache.ignite.cache.QueryIndexType;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.database.DbCheckpointListener;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
-import org.apache.ignite.internal.util.lang.GridAbsPredicate;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
-
-/**
- *
- */
-public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Index name. */
- private static final String IDX_NAME = "my_idx";
-
- /** Cache name. */
- private static final String TMPL_NAME = "test_cache*";
-
- /** Table name. */
- private static final String TBL_NAME = Person.class.getSimpleName();
-
- /** Schema name. */
- private static final String SCHEMA_NAME = "PUBLIC";
-
- /** Cache name. */
- private static final String CACHE_NAME = TBL_NAME;
-
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
-
- cfg.setCacheConfiguration(cacheCfg(TMPL_NAME));
-
- PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
-
- pCfg.setCheckpointingFrequency(1000);
-
- cfg.setPersistentStoreConfiguration(pCfg);
-
- return cfg;
- }
-
- /** */
- private CacheConfiguration cacheCfg(String name) {
- CacheConfiguration<?, ?> cfg = new CacheConfiguration<>();
-
- cfg.setName(name);
-
- cfg.setRebalanceMode(CacheRebalanceMode.NONE);
-
- cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
-
- return cfg;
- }
-
- /** */
- private QueryEntity getEntity() {
- LinkedHashMap<String, String> fields = new LinkedHashMap<>();
-
- fields.put("id", Integer.class.getName());
- fields.put("name", String.class.getName());
-
- QueryEntity entity = new QueryEntity(Integer.class.getName(), Person.class.getName());
- entity.setFields(fields);
- entity.setTableName(TBL_NAME);
-
- return entity;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- System.setProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK, "true");
-
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
-
- System.clearProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK);
- }
-
- /** */
- public void testPersistIndex() throws Exception {
- IgniteEx ig0 = startGrid(0);
- startGrid(1);
-
- final AtomicInteger cnt = new AtomicInteger();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
-
- db.addCheckpointListener(new DbCheckpointListener() {
- @Override public void onCheckpointBegin(Context context) {
- cnt.incrementAndGet();
- }
- });
-
- QueryIndex idx = new QueryIndex("name");
-
- idx.setName(IDX_NAME);
-
- ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, 1, true);
-
- assert indexCnt(ig0, CACHE_NAME) == 0;
-
- ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get();
-
- assert indexCnt(ig0, CACHE_NAME) == 1;
-
- waitForCheckpoint(cnt);
-
- stopGrid(1);
-
- IgniteEx ig1 = startGrid(1);
-
- assert indexCnt(ig1, CACHE_NAME) == 1;
- }
-
- /** */
- public void testPersistCompositeIndex() throws Exception {
- IgniteEx ig0 = startGrid(0);
- startGrid(1);
-
- final AtomicInteger cnt = new AtomicInteger();
-
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
-
- db.addCheckpointListener(new DbCheckpointListener() {
- @Override public void onCheckpointBegin(Context context) {
- cnt.incrementAndGet();
- }
- });
-
- ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, 1, true);
-
- assert indexCnt(ig0, CACHE_NAME) == 0;
-
- QueryIndex idx = new QueryIndex(Arrays.asList("id", "name"), QueryIndexType.SORTED);
-
- idx.setName(IDX_NAME);
-
- ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get();
-
- assert indexCnt(ig0, CACHE_NAME) == 1;
-
- waitForCheckpoint(cnt);
-
- stopGrid(1);
-
- IgniteEx ig1 = startGrid(1);
-
- assert indexCnt(ig1, CACHE_NAME) == 1;
- }
-
- /** */
- private void waitForCheckpoint(final AtomicInteger cnt) throws IgniteInterruptedCheckedException {
- final int i = cnt.get();
-
- GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return cnt.get() > i;
- }
- }, 2000);
- }
-
- /** */
- private int indexCnt(IgniteEx node, String cacheName) {
-
- DynamicCacheDescriptor desc = node.context().cache().cacheDescriptor(cacheName);
-
- int cnt = 0;
-
- for (QueryEntity entity : desc.schema().entities()) {
- cnt += entity.getIndexes().size();
- }
-
- return cnt;
- }
-
- /**
- * @return Indexing.
- */
- private IgniteH2Indexing getIndexing(IgniteEx ignite) {
- return U.field(ignite.context().query(), "idx");
- }
-
- /**
- *
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- *
- */
- protected static class Person implements Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- @SuppressWarnings("unused")
- private Person() {
- // No-op.
- }
-
- /** */
- public Person(int id) {
- this.id = id;
- }
-
- /** */
- @QuerySqlField
- protected int id;
-
- /** */
- @QuerySqlField
- protected String name;
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- IgnitePersistentStoreSchemaLoadTest.Person person = (IgnitePersistentStoreSchemaLoadTest.Person)o;
-
- if (id != person.id)
- return false;
-
- return name != null ? name.equals(person.name) : person.name == null;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int result = id;
-
- result = 31 * result + (name != null ? name.hashCode() : 0);
-
- return result;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsMultiNodePutGetRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsMultiNodePutGetRestartTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsMultiNodePutGetRestartTest.java
deleted file mode 100644
index 41b1b29..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsMultiNodePutGetRestartTest.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db;
-
-import java.io.File;
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.configuration.MemoryConfiguration;
-
-/**
- *
- */
-public class IgnitePdsMultiNodePutGetRestartTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final int GRID_CNT = 3;
-
- /** */
- private static final String CACHE_NAME = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
-
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
- dbCfg.setMemoryPolicies(memPlcCfg);
-
- cfg.setMemoryConfiguration(dbCfg);
-
- CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
-
- ccfg.setIndexedTypes(Integer.class, DbValue.class);
-
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
-
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- cfg.setCacheConfiguration(ccfg);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration bCfg = new BinaryConfiguration();
-
- bCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(bCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPutGetSimple() throws Exception {
- String home = U.getIgniteHome();
-
- File allocPath = new File(home, "work/db/" + UUID.randomUUID());
-
- allocPath.mkdirs();
-
- info(">>> Will use path: " + allocPath);
-
- startGrids(GRID_CNT);
-
- try {
- IgniteEx ig = grid(0);
-
- checkPutGetSql(ig, true);
- }
- finally {
- stopAllGrids();
- }
-
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
-
- startGrids(GRID_CNT);
-
- try {
- IgniteEx ig = grid(0);
-
- checkPutGetSql(ig, false);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param ig Ig.
- * @param write Write.
- */
- private void checkPutGetSql(IgniteEx ig, boolean write) {
- IgniteCache<Integer, DbValue> cache = ig.cache(CACHE_NAME);
-
- if (write) {
- try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(CACHE_NAME)) {
- for (int i = 0; i < 10_000; i++)
- streamer.addData(i, new DbValue(i, "value-" + i, i));
- }
- }
-
- List<List<?>> res = cache.query(new SqlFieldsQuery("select ival from dbvalue where ival < ? order by ival asc")
- .setArgs(10_000)).getAll();
-
- assertEquals(10_000, res.size());
-
- for (int i = 0; i < 10_000; i++) {
- assertEquals(1, res.get(i).size());
- assertEquals(i, res.get(i).get(0));
- }
-
- assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
- assertEquals(5000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 10000"))
- .getAll().size());
-
- for (int i = 0; i < 10_000; i++)
- assertEquals(new DbValue(i, "value-" + i, i), cache.get(i));
- }
-
- /**
- *
- */
- private static class DbValue implements Serializable {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String sVal;
-
- /** */
- @QuerySqlField
- private long lVal;
-
- /**
- * @param iVal Integer value.
- * @param sVal String value.
- * @param lVal Long value.
- */
- public DbValue(int iVal, String sVal, long lVal) {
- this.iVal = iVal;
- this.sVal = sVal;
- this.lVal = lVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- DbValue dbVal = (DbValue)o;
-
- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(DbValue.class, this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionDuringPartitionClearTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionDuringPartitionClearTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionDuringPartitionClearTest.java
deleted file mode 100644
index b4ffde8..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionDuringPartitionClearTest.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db;
-
-import java.util.concurrent.Callable;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsPageEvictionDuringPartitionClearTest extends GridCommonAbstractTest {
- /** */
- public static final String CACHE_NAME = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME)
- .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
- .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
- .setAffinity(new RendezvousAffinityFunction(false, 128))
- .setRebalanceMode(CacheRebalanceMode.SYNC)
- .setBackups(1);
-
- cfg.setCacheConfiguration(ccfg);
-
- // Intentionally set small page cache size.
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setInitialSize(70 * 1024 * 1024);
- memPlcCfg.setMaxSize(70 * 1024 * 1024);
-
- memPlcCfg.setName("dfltMemPlc");
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- memCfg.setMemoryPolicies(memPlcCfg);
-
- memCfg.setDefaultMemoryPolicyName(memPlcCfg.getName());
-
- cfg.setMemoryConfiguration(memCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return 20 * 60 * 1000;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC, "true");
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageEvictionOnNodeStart() throws Exception {
- for (int r = 0; r < 3; r++) {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- startGrids(2);
-
- try {
- Ignite ig = ignite(0);
-
- IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(CACHE_NAME);
-
- for (int i = 0; i < 300_000; i++) {
- streamer.addData(i, new TestValue(i));
-
- if (i > 0 && i % 10_000 == 0)
- info("Done: " + i);
- }
-
- streamer.flush();
-
- IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteEx ig = startGrid(2);
-
- info(">>>>>>>>>>>");
- info(">>>>>>>>>>>");
- info(">>>>>>>>>>>");
-
- return ig;
- }
- });
-
- for (int i = 500_000; i < 1_000_000; i++) {
- streamer.addData(i, new TestValue(i));
-
- if (i > 0 && i % 10_000 == 0) {
- info("Done: " + i);
-
- U.sleep(1000);
- }
- }
-
- streamer.close();
-
- fut.get();
- }
- finally {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
- }
- }
-
- /**
- *
- */
- private static class TestValue {
- /** */
- private int id;
-
- /** */
- private byte[] payload = new byte[512];
-
- /**
- * @param id ID.
- */
- private TestValue(int id) {
- this.id = id;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionTest.java
deleted file mode 100644
index d48617d..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsPageEvictionTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db;
-
-import java.io.Serializable;
-import java.util.List;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsPageEvictionTest extends GridCommonAbstractTest {
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Test entry count. */
- public static final int ENTRY_CNT = 1_000_000;
-
- /** Cache name. */
- private static final String CACHE_NAME = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- memCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-
- memCfg.setPageSize(1024);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(50 * 1024 * 1024);
- memPlcCfg.setMaxSize(50 * 1024 * 1024);
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- CacheConfiguration<DbKey, DbValue> ccfg = new CacheConfiguration<>(CACHE_NAME);
-
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
- ccfg.setIndexedTypes(DbKey.class, DbValue.class);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- cfg.setCacheConfiguration(ccfg);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
-
- cfg.setDiscoverySpi(
- new TcpDiscoverySpi()
- .setIpFinder(IP_FINDER)
- );
-
- cfg.setMarshaller(null);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- stopAllGrids();
-
- startGrids(1);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageEvictionSql() throws Exception {
- IgniteEx ig = grid(0);
-
- try (IgniteDataStreamer<DbKey, DbValue> streamer = ig.dataStreamer(CACHE_NAME)) {
- for (int i = 0; i < ENTRY_CNT; i++) {
- streamer.addData(new DbKey(i), new DbValue(i, "value-" + i, Long.MAX_VALUE - i));
-
- if (i > 0 && i % 10_000 == 0)
- info("Done put: " + i);
- }
- }
-
- IgniteCache<DbKey, DbValue> cache = ignite(0).cache(CACHE_NAME);
-
- for (int i = 0; i < ENTRY_CNT; i++) {
- assertEquals(Long.MAX_VALUE - i, cache.get(new DbKey(i)).lVal);
-
- if (i > 0 && i % 10_000 == 0)
- info("Done get: " + i);
- }
-
- for (int i = 0; i < ENTRY_CNT; i++) {
- List<List<?>> rows = cache.query(
- new SqlFieldsQuery("select lVal from DbValue where iVal=?").setArgs(i)
- ).getAll();
-
- assertEquals(1, rows.size());
- assertEquals(Long.MAX_VALUE - i, rows.get(0).get(0));
-
- if (i > 0 && i % 10_000 == 0)
- info("Done SQL query: " + i);
- }
- }
-
- /**
- *
- */
- private static class DbKey implements Serializable {
- /** */
- private int val;
-
- /**
- * @param val Value.
- */
- private DbKey(int val) {
- this.val = val;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || !(o instanceof DbKey))
- return false;
-
- DbKey key = (DbKey)o;
-
- return val == key.val;
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return val;
- }
- }
-
- /**
- *
- */
- private static class DbValue implements Serializable {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String sVal;
-
- /** */
- @QuerySqlField
- private long lVal;
-
- /**
- * @param iVal Integer value.
- * @param sVal String value.
- * @param lVal Long value.
- */
- private DbValue(int iVal, String sVal, long lVal) {
- this.iVal = iVal;
- this.sVal = sVal;
- this.lVal = lVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- DbValue dbVal = (DbValue)o;
-
- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(DbValue.class, this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsRebalancingOnNotStableTopologyTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsRebalancingOnNotStableTopologyTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsRebalancingOnNotStableTopologyTest.java
deleted file mode 100644
index dfe174a..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsRebalancingOnNotStableTopologyTest.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db;
-
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.PartitionLossPolicy;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
-
-/**
- * We start writing to unstable cluster.
- * After that we start killing node.
- * There will be entries in WAL which belongs to evicted partitions.
- * We should ignore them (not throw exceptions). This point is tested.
- */
-public class IgnitePdsRebalancingOnNotStableTopologyTest extends GridCommonAbstractTest {
- /** Checkpoint frequency. */
- private static final long CHECKPOINT_FREQUENCY = 2_000_000;
-
- /** Cluster size. */
- private static final int CLUSTER_SIZE = 5;
-
- /** */
- private static final String CACHE_NAME = "cache1";
-
- /**
- * @throws Exception When fails.
- */
- public void test() throws Exception {
- Ignite ex = startGrid(0);
-
- startGrid(1);
-
- final CountDownLatch startLatch = new CountDownLatch(1);
- final CountDownLatch doneLatch = new CountDownLatch(1);
-
- final Ignite ex1 = ex;
-
- final AtomicBoolean stop = new AtomicBoolean();
- final AtomicInteger keyCnt = new AtomicInteger();
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- ex1.active(true);
-
- try {
- checkTopology(2);
-
- startLatch.countDown();
-
- IgniteCache<Object, Object> cache1 = ex1.cache(CACHE_NAME);
-
- int key = keyCnt.get();
-
- while (!stop.get()) {
- if (key > 0 && (key % 500 == 0)) {
- U.sleep(5);
-
- System.out.println("key = " + key);
- }
-
- cache1.put(key, -key);
-
- key = keyCnt.incrementAndGet();
- }
- }
- catch (Throwable th) {
- th.printStackTrace();
- }
-
- doneLatch.countDown();
- }
- });
-
- thread.setName("Data-Loader");
- thread.start();
-
- startLatch.await(60, TimeUnit.SECONDS);
-
- for (int i = 2; i < CLUSTER_SIZE; i++) {
- startGrid(i);
-
- U.sleep(5000);
- }
-
- U.sleep(10000);
-
- IgniteProcessProxy.kill("db.RebalancingOnNotStableTopologyTest2");
-
- Thread.sleep(5000);
-
- IgniteProcessProxy.kill("db.RebalancingOnNotStableTopologyTest1");
-
- assert doneLatch.getCount() > 0;
-
- stop.set(true);
-
- doneLatch.await(600, TimeUnit.SECONDS);
-
- IgniteProcessProxy.killAll();
-
- stopAllGrids();
-
- //start cluster. it will cause memory restoration and reading WAL.
- ex = startGrids(CLUSTER_SIZE);
-
- ex.active(true);
-
- checkTopology(CLUSTER_SIZE);
-
- IgniteCache<Object, Object> cache1 = ex.cache(CACHE_NAME);
-
- assert keyCnt.get() > 0;
-
- for (int i = 0; i < keyCnt.get(); i++)
- assertEquals(-i, cache1.get(i));
-
- System.out.println("Test finished with total keys count = " + keyCnt.get());
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setActiveOnStart(false);
-
- CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>(CACHE_NAME);
-
- ccfg.setPartitionLossPolicy(PartitionLossPolicy.READ_ONLY_SAFE);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setCacheMode(CacheMode.PARTITIONED);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
- ccfg.setBackups(2);
-
- cfg.setCacheConfiguration(ccfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setCheckpointingFrequency(CHECKPOINT_FREQUENCY)
- );
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(200 * 1024 * 1024);
- memPlcCfg.setMaxSize(200 * 1024 * 1024);
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean isMultiJvm() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override protected boolean checkTopology() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return TimeUnit.MINUTES.toMillis(10);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsTransactionsHangTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsTransactionsHangTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsTransactionsHangTest.java
deleted file mode 100644
index 0007a3b..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsTransactionsHangTest.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db;
-
-import java.util.Random;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.TransactionConfiguration;
-import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.apache.ignite.transactions.Transaction;
-import org.apache.ignite.transactions.TransactionIsolation;
-import org.jsr166.LongAdder8;
-
-import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
-
-/**
- * Checks that transactions don't hang during checkpoint creation.
- */
-public class IgnitePdsTransactionsHangTest extends GridCommonAbstractTest {
- /** IP finder. */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** Page cache size. */
- private static final int PAGE_CACHE_SIZE = 512;
-
- /** Page size. */
- private static final Integer PAGE_SIZE = 16;
-
- /** Cache name. */
- private static final String CACHE_NAME = "IgnitePdsTransactionsHangTest";
-
- /** Number of insert threads. */
- public static final int THREADS_CNT = 32;
-
- /** Warm up period, seconds. */
- public static final int WARM_UP_PERIOD = 20;
-
- /** Duration. */
- public static final int DURATION = 180;
-
- /** Maximum count of inserted keys. */
- public static final int MAX_KEY_COUNT = 500_000;
-
- /** Checkpoint frequency. */
- public static final long CHECKPOINT_FREQUENCY = 20_000;
-
- /** {@inheritDoc} */
- @Override protected long getTestTimeout() {
- return Long.MAX_VALUE;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
- discoSpi.setIpFinder(IP_FINDER);
- cfg.setDiscoverySpi(discoSpi);
-
- BinaryConfiguration binaryCfg = new BinaryConfiguration();
- binaryCfg.setCompactFooter(false);
- cfg.setBinaryConfiguration(binaryCfg);
-
- cfg.setPeerClassLoadingEnabled(true);
-
- TcpCommunicationSpi tcpCommSpi = new TcpCommunicationSpi();
-
- tcpCommSpi.setSharedMemoryPort(-1);
- cfg.setCommunicationSpi(tcpCommSpi);
-
- TransactionConfiguration txCfg = new TransactionConfiguration();
-
- txCfg.setDefaultTxIsolation(TransactionIsolation.READ_COMMITTED);
-
- cfg.setTransactionConfiguration(txCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalHistorySize(1)
- .setCheckpointingFrequency(CHECKPOINT_FREQUENCY)
- );
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(PAGE_CACHE_SIZE * 1024 * 1024);
- memPlcCfg.setMaxSize(PAGE_CACHE_SIZE * 1024 * 1024);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- memCfg.setPageSize(PAGE_SIZE * 1024);
-
- cfg.setMemoryConfiguration(memCfg);
-
- return cfg;
- }
-
- /**
- * Creates cache configuration.
- *
- * @return Cache configuration.
- * */
- private CacheConfiguration getCacheConfiguration() {
- CacheConfiguration ccfg = new CacheConfiguration();
-
- ccfg.setName(CACHE_NAME);
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4));
- ccfg.setReadFromBackup(true);
-
- ccfg.setCacheMode(CacheMode.PARTITIONED);
-
- return ccfg;
- }
-
- /**
- * Copied from customers benchmark.
- *
- * @throws Exception If failed.
- * */
- public void testTransactionsDontHang() throws Exception {
- try {
- final Ignite g = startGrids(2);
-
- g.getOrCreateCache(getCacheConfiguration());
-
- ExecutorService threadPool = Executors.newFixedThreadPool(THREADS_CNT);
- final CyclicBarrier cyclicBarrier = new CyclicBarrier(THREADS_CNT);
-
- final AtomicBoolean interrupt = new AtomicBoolean(false);
- final LongAdder8 operationCnt = new LongAdder8();
-
- final IgniteCache<Long, TestEntity> cache = g.cache(CACHE_NAME);
-
- for (int i = 0; i < THREADS_CNT; i++) {
- threadPool.submit(new Runnable() {
- @Override public void run() {
- try {
- ThreadLocalRandom locRandom = ThreadLocalRandom.current();
-
- cyclicBarrier.await();
-
- while (!interrupt.get()) {
- long randomKey = locRandom.nextLong(MAX_KEY_COUNT);
-
- TestEntity entity = TestEntity.newTestEntity(locRandom);
-
- try (Transaction tx = g.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
- cache.put(randomKey, entity);
-
- tx.commit();
- }
-
- operationCnt.increment();
- }
- }
- catch (Throwable e) {
- System.out.println(e.toString());
-
- throw new RuntimeException(e);
- }
- }
- });
- }
-
- long stopTime = System.currentTimeMillis() + DURATION * 1000;
- long totalOperations = 0;
- int periods = 0;
- long max = Long.MIN_VALUE, min = Long.MAX_VALUE;
-
- while (System.currentTimeMillis() < stopTime) {
- U.sleep(1000);
-
- long sum = operationCnt.sumThenReset();
- periods++;
-
- if (periods > WARM_UP_PERIOD) {
- totalOperations += sum;
-
- max = Math.max(max, sum);
- min = Math.min(min, sum);
-
- System.out.println("Operation count: " + sum + " min=" + min + " max=" + max + " avg=" + totalOperations / (periods - WARM_UP_PERIOD));
- }
- }
-
- interrupt.set(true);
-
- threadPool.shutdown();
- System.out.println("Test complete");
-
- threadPool.awaitTermination(getTestTimeout(), TimeUnit.MILLISECONDS);
-
- IgniteTxManager tm = internalCache(cache).context().tm();
-
- assertEquals("There are still active transactions", 0, tm.activeTransactions().size());
- } finally {
- stopAllGrids();
- }
- }
-
- /**
- * Entity for test.
- * */
- public static class TestEntity {
- /** String value. */
- private String strVal;
-
- /** Long value. */
- private Long longVal;
-
- /** Int value. */
- private int intVal;
-
- /**
- * @param strVal String value.
- */
- public void setStrVal(String strVal) {
- this.strVal = strVal;
- }
-
- /**
- * @param longVal Long value.
- */
- public void setLongVal(Long longVal) {
- this.longVal = longVal;
- }
-
- /**
- * @param intVal Integer value.
- */
- public void setIntVal(int intVal) {
- this.intVal = intVal;
- }
-
- /**
- * Creates test entity with random values.
- *
- * @param random Random seq generator.
- * @return new test entity
- * */
- private static TestEntity newTestEntity(Random random) {
- TestEntity entity = new TestEntity();
-
- entity.setLongVal((long) random.nextInt(1_000));
- entity.setIntVal(random.nextInt(1_000));
- entity.setStrVal("test" + random.nextInt(1_000));
-
- return entity;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsWholeClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsWholeClusterRestartTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsWholeClusterRestartTest.java
deleted file mode 100644
index 576b40e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/IgnitePdsWholeClusterRestartTest.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.checkpoint.noop.NoopCheckpointSpi;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsWholeClusterRestartTest extends GridCommonAbstractTest {
- /** */
- private static final int GRID_CNT = 5;
-
- /** */
- private static final int ENTRIES_COUNT = 1_000;
-
- /** */
- public static final String CACHE_NAME = "cache1";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(dbCfg);
-
- CacheConfiguration ccfg1 = new CacheConfiguration();
-
- ccfg1.setName(CACHE_NAME);
- ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg1.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
- ccfg1.setBackups(2);
-
- cfg.setLateAffinityAssignment(false);
- cfg.setActiveOnStart(false);
-
- // To avoid hostname lookup on start.
- cfg.setCheckpointSpi(new NoopCheckpointSpi());
-
- cfg.setCacheConfiguration(ccfg1);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setConsistentId(gridName);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRestarts() throws Exception {
- startGrids(GRID_CNT);
-
- ignite(0).active(true);
-
- awaitPartitionMapExchange();
-
- try (IgniteDataStreamer<Object, Object> ds = ignite(0).dataStreamer(CACHE_NAME)) {
- for (int i = 0; i < ENTRIES_COUNT; i++)
- ds.addData(i, i);
- }
-
- stopAllGrids();
-
- List<Integer> idxs = new ArrayList<>();
-
- for (int i = 0; i < GRID_CNT; i++)
- idxs.add(i);
-
- for (int r = 0; r < 10; r++) {
- Collections.shuffle(idxs);
-
- info("Will start in the following order: " + idxs);
-
- for (Integer idx : idxs)
- startGrid(idx);
-
- try {
- ignite(0).active(true);
-
- for (int g = 0; g < GRID_CNT; g++) {
- Ignite ig = ignite(g);
-
- for (int k = 0; k < ENTRIES_COUNT; k++)
- assertEquals("Failed to read [g=" + g + ", part=" + ig.affinity(CACHE_NAME).partition(k) +
- ", nodes=" + ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(k) + ']',
- k, ig.cache(CACHE_NAME).get(k));
- }
- }
- finally {
- stopAllGrids();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsCacheIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsCacheIntegrationTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsCacheIntegrationTest.java
deleted file mode 100644
index 97bb05e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsCacheIntegrationTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.file;
-
-import java.io.Serializable;
-import java.util.List;
-import java.util.concurrent.Callable;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsCacheIntegrationTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final int GRID_CNT = 3;
-
- /** */
- private static final String CACHE_NAME = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setMemoryConfiguration(dbCfg);
-
- CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
-
- ccfg.setIndexedTypes(Integer.class, DbValue.class);
-
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
-
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- cfg.setCacheConfiguration(ccfg);
-
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration bCfg = new BinaryConfiguration();
-
- bCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(bCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPutGetSimple() throws Exception {
- startGrids(GRID_CNT);
-
- try {
- IgniteEx ig = grid(0);
-
- checkPutGetSql(ig, true);
- }
- finally {
- stopAllGrids();
- }
-
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
-
- startGrids(GRID_CNT);
-
- try {
- IgniteEx ig = grid(0);
-
- checkPutGetSql(ig, false);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPutMultithreaded() throws Exception {
- startGrids(4);
-
- try {
- final IgniteEx grid = grid(0);
-
- GridTestUtils.runMultiThreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- for (int i = 0; i < 1000; i++)
- grid.cache(CACHE_NAME).put(i, i);
-
- return null;
- }
- }, 8, "updater");
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param ig Ignite instance.
- * @param write Write flag.
- */
- private void checkPutGetSql(Ignite ig, boolean write) {
- IgniteCache<Integer, DbValue> cache = ig.cache(CACHE_NAME);
-
- int entryCnt = 50_000;
-
- if (write) {
- try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(CACHE_NAME)) {
- streamer.allowOverwrite(true);
-
- for (int i = 0; i < entryCnt; i++)
- streamer.addData(i, new DbValue(i, "value-" + i, i));
- }
- }
-
- for (int i = 0; i < GRID_CNT; i++) {
- IgniteEx ignite = grid(i);
-
- GridCacheAdapter<Object, Object> cache0 = ignite.context().cache().internalCache(CACHE_NAME);
-
- for (int k = 0; k < entryCnt; k++)
- assertNull(cache0.peekEx(i));
-
- assertEquals(entryCnt, ignite.cache(CACHE_NAME).size());
- }
-
- for (int i = 0; i < entryCnt; i++)
- assertEquals("i = " + i, new DbValue(i, "value-" + i, i), cache.get(i));
-
- List<List<?>> res = cache.query(new SqlFieldsQuery("select ival from dbvalue where ival < ? order by ival asc")
- .setArgs(10_000)).getAll();
-
- assertEquals(10_000, res.size());
-
- for (int i = 0; i < 10_000; i++) {
- assertEquals(1, res.get(i).size());
- assertEquals(i, res.get(i).get(0));
- }
-
- assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
- assertEquals(5000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 10000"))
- .getAll().size());
-
- for (int i = 0; i < 10_000; i++)
- assertEquals(new DbValue(i, "value-" + i, i), cache.get(i));
- }
-
- /**
- *
- */
- private static class DbValue implements Serializable {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String sVal;
-
- /** */
- @QuerySqlField
- private long lVal;
-
- /**
- * @param iVal Integer value.
- * @param sVal String value.
- * @param lVal Long value.
- */
- private DbValue(int iVal, String sVal, long lVal) {
- this.iVal = iVal;
- this.sVal = sVal;
- this.lVal = lVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- DbValue dbVal = (DbValue)o;
-
- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(DbValue.class, this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsEvictionTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsEvictionTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsEvictionTest.java
deleted file mode 100644
index 45bcef6..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/db/file/IgnitePdsEvictionTest.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.db.file;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.concurrent.Callable;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- * Test for page evictions.
- */
-public class IgnitePdsEvictionTest extends GridCommonAbstractTest {
- /** */
- private static final int NUMBER_OF_SEGMENTS = 64;
-
- /** */
- private static final int PAGE_SIZE = 1024;
-
- /** */
- private static final long CHUNK_SIZE = 1024 * 1024;
-
- /** */
- private static final long MEMORY_LIMIT = 10 * CHUNK_SIZE;
-
- /** */
- private static final int PAGES_NUM = 128_000;
-
- /** Cache name. */
- private final String cacheName = "cache";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- final IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
-
- cfg.setMemoryConfiguration(createDbConfig());
-
- cfg.setCacheConfiguration(new CacheConfiguration<>(cacheName));
-
- return cfg;
- }
-
- /**
- * @return DB config.
- */
- private MemoryConfiguration createDbConfig() {
- final MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
- memPlcCfg.setInitialSize(MEMORY_LIMIT);
- memPlcCfg.setMaxSize(MEMORY_LIMIT);
- memPlcCfg.setName("dfltMemPlc");
-
- memCfg.setPageSize(PAGE_SIZE);
- memCfg.setConcurrencyLevel(NUMBER_OF_SEGMENTS);
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- return memCfg;
- }
-
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /**
- * @throws Exception If fail.
- */
- public void testPageEviction() throws Exception {
- final IgniteEx ig = startGrid(0);
-
- final PageMemory memory = getMemory(ig);
-
- writeData(ig, memory, CU.cacheId(cacheName));
- }
-
- /**
- * @param memory Page memory.
- * @param cacheId Cache id.
- * @throws IgniteCheckedException If failed.
- */
- private void writeData(final IgniteEx ignite, final PageMemory memory, final int cacheId) throws Exception {
- final int size = PAGES_NUM;
-
- final List<FullPageId> pageIds = new ArrayList<>(size);
-
- IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
-
- // Allocate.
- for (int i = 0; i < size; i++) {
- db.checkpointReadLock();
- try {
- final FullPageId fullId = new FullPageId(memory.allocatePage(cacheId, i % 256, PageMemory.FLAG_DATA),
- cacheId);
-
- pageIds.add(fullId);
- }
- finally {
- db.checkpointReadUnlock();
- }
- }
-
- System.out.println("Allocated pages: " + pageIds.size());
-
- // Write data. (Causes evictions.)
- final int part = PAGES_NUM / NUMBER_OF_SEGMENTS;
-
- final Collection<IgniteInternalFuture> futs = new ArrayList<>();
-
- for (int i = 0; i < PAGES_NUM; i += part)
- futs.add(runWriteInThread(ignite, i, i + part, memory, pageIds));
-
- for (final IgniteInternalFuture fut : futs)
- fut.get();
-
- System.out.println("Wrote pages: " + pageIds.size());
-
- // Read data. (Causes evictions.)
- futs.clear();
-
- for (int i = 0; i < PAGES_NUM; i += part)
- futs.add(runReadInThread(ignite, i, i + part, memory, pageIds));
-
- for (final IgniteInternalFuture fut : futs)
- fut.get();
-
- System.out.println("Read pages: " + pageIds.size());
- }
-
- /**
- * @param start Start index.
- * @param end End index.
- * @param memory PageMemory.
- * @param pageIds Allocated pages.
- * @return Future.
- * @throws Exception If fail.
- */
- private IgniteInternalFuture runWriteInThread(
- final IgniteEx ignite,
- final int start,
- final int end,
- final PageMemory memory,
- final List<FullPageId> pageIds
- ) throws Exception {
-
- return GridTestUtils.runAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
-
- for (int i = start; i < end; i++) {
- db.checkpointReadLock();
-
- try {
- FullPageId fullId = pageIds.get(i);
-
- long page = memory.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- final long pageAddr = memory.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- try {
- PageIO.setPageId(pageAddr, fullId.pageId());
-
- PageUtils.putLong(pageAddr, PageIO.COMMON_HEADER_END, i * 2);
- }
- finally {
- memory.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
- }
- }
- finally {
- memory.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- finally {
- db.checkpointReadUnlock();
- }
- }
-
- return null;
- }
- });
- }
-
- /**
- * @param start Start index.
- * @param end End index.
- * @param memory PageMemory.
- * @param pageIds Allocated pages.
- * @return Future.
- * @throws Exception If fail.
- */
- private IgniteInternalFuture runReadInThread(final IgniteEx ignite, final int start, final int end,
- final PageMemory memory,
- final List<FullPageId> pageIds) throws Exception {
- return GridTestUtils.runAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
-
- for (int i = start; i < end; i++) {
- db.checkpointReadLock();
-
- try {
- final FullPageId fullId = pageIds.get(i);
-
- long page = memory.acquirePage(fullId.cacheId(), fullId.pageId());
- try {
- final long pageAddr = memory.readLock(fullId.cacheId(), fullId.pageId(), page);
-
- try {
- assertEquals(i * 2, PageUtils.getLong(pageAddr, PageIO.COMMON_HEADER_END));
- }
- finally {
- memory.readUnlock(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- finally {
- memory.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- finally {
- db.checkpointReadUnlock();
- }
- }
-
- return null;
- }
- });
- }
-
- /**
- * @param ig Ignite instance.
- * @return Memory and store.
- * @throws Exception If failed to initialize the store.
- */
- private PageMemory getMemory(IgniteEx ig) throws Exception {
- final GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
-
- final IgniteCacheDatabaseSharedManager db = sharedCtx.database();
-
- return db.memoryPolicy(null).pageMemory();
- }
-
- /**
- * @throws IgniteCheckedException If fail.
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
[39/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
new file mode 100755
index 0000000..2146f98
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -0,0 +1,3113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardOpenOption;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import javax.management.InstanceNotFoundException;
+import javax.management.JMException;
+import javax.management.MBeanRegistrationException;
+import javax.management.ObjectName;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.PersistenceMetrics;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.DataPageEvictionMode;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.events.DiscoveryEvent;
+import org.apache.ignite.events.EventType;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.NodeStoppingException;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CacheState;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.ClusterState;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.CheckpointMetricsTracker;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PagePartitionMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
+import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
+import org.apache.ignite.internal.processors.port.GridPortRecord;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.future.CountDownFuture;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.tostring.GridToStringInclude;
+import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.P3;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.internal.util.worker.GridWorker;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.mxbean.PersistenceMetricsMXBean;
+import org.apache.ignite.thread.IgniteThread;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+
+/**
+ *
+ */
+@SuppressWarnings({"unchecked", "NonPrivateFieldAccessedInSynchronizedContext"})
+public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedManager {
+ /** */
+ public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC";
+
+ /** Skip sync. */
+ private final boolean skipSync = IgniteSystemProperties.getBoolean(IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
+
+ /** */
+ private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
+
+ /** */
+ private final int walRebalanceThreshold = IgniteSystemProperties.getInteger(
+ IGNITE_PDS_WAL_REBALANCE_THRESHOLD, 500_000);
+
+ /** Checkpoint lock hold count. */
+ private static final ThreadLocal<Integer> CHECKPOINT_LOCK_HOLD_COUNT = new ThreadLocal<Integer>() {
+ @Override protected Integer initialValue() {
+ return 0;
+ }
+ };
+
+ /** Assertion enabled. */
+ private static final boolean ASSERTION_ENABLED = GridCacheDatabaseSharedManager.class.desiredAssertionStatus();
+
+ /** Checkpoint file name pattern. */
+ private static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
+
+ /** */
+ private static final FileFilter CP_FILE_FILTER = new FileFilter() {
+ @Override public boolean accept(File f) {
+ return CP_FILE_NAME_PATTERN.matcher(f.getName()).matches();
+ }
+ };
+
+ /** */
+ private static final Comparator<GridDhtLocalPartition> ASC_PART_COMPARATOR = new Comparator<GridDhtLocalPartition>() {
+ @Override public int compare(GridDhtLocalPartition a, GridDhtLocalPartition b) {
+ return Integer.compare(a.id(), b.id());
+ }
+ };
+
+ /** */
+ private static final Comparator<File> CP_TS_COMPARATOR = new Comparator<File>() {
+ /** {@inheritDoc} */
+ @Override public int compare(File o1, File o2) {
+ Matcher m1 = CP_FILE_NAME_PATTERN.matcher(o1.getName());
+ Matcher m2 = CP_FILE_NAME_PATTERN.matcher(o2.getName());
+
+ boolean s1 = m1.matches();
+ boolean s2 = m2.matches();
+
+ assert s1 : "Failed to match CP file: " + o1.getAbsolutePath();
+ assert s2 : "Failed to match CP file: " + o2.getAbsolutePath();
+
+ long ts1 = Long.parseLong(m1.group(1));
+ long ts2 = Long.parseLong(m2.group(1));
+
+ int res = Long.compare(ts1, ts2);
+
+ if (res == 0) {
+ CheckpointEntryType type1 = CheckpointEntryType.valueOf(m1.group(3));
+ CheckpointEntryType type2 = CheckpointEntryType.valueOf(m2.group(3));
+
+ assert type1 != type2 : "o1=" + o1.getAbsolutePath() + ", o2=" + o2.getAbsolutePath();
+
+ res = type1 == CheckpointEntryType.START ? -1 : 1;
+ }
+
+ return res;
+ }
+ };
+
+ /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */
+ private volatile Checkpointer checkpointer;
+
+ /** For testing only. */
+ private volatile boolean checkpointsEnabled = true;
+
+ /** For testing only. */
+ private volatile GridFutureAdapter<Void> enableChangeApplied;
+
+ /** */
+ private ReentrantReadWriteLock checkpointLock = new ReentrantReadWriteLock();
+
+ /** */
+ private long checkpointFreq;
+
+ /** */
+ private long checkpointPageBufSize;
+
+ /** */
+ private FilePageStoreManager storeMgr;
+
+ /** */
+ private File cpDir;
+
+ /** */
+ private volatile boolean printCheckpointStats = true;
+
+ /** Database configuration. */
+ private final PersistentStoreConfiguration persistenceCfg;
+
+ /** */
+ private final Collection<DbCheckpointListener> lsnrs = new CopyOnWriteArrayList<>();
+
+ /** Checkpoint history. */
+ private final CheckpointHistory checkpointHist = new CheckpointHistory();
+
+ /** */
+ private boolean stopping;
+
+ /** Checkpoint runner thread pool. */
+ private ExecutorService asyncRunner;
+
+ /** Buffer for the checkpoint threads. */
+ private ThreadLocal<ByteBuffer> threadBuf;
+
+ /** */
+ private final ConcurrentMap<Integer, IgniteInternalFuture> idxRebuildFuts = new ConcurrentHashMap<>();
+
+ /** Lock holder. */
+ private FileLockHolder fileLockHolder;
+
+ /** Lock wait time. */
+ private final int lockWaitTime;
+
+ /** */
+ private Map<Integer, Map<Integer, T2<Long, WALPointer>>> reservedForExchange;
+
+ /** */
+ private final ConcurrentMap<T2<Integer, Integer>, T2<Long, WALPointer>> reservedForPreloading = new ConcurrentHashMap<>();
+
+ /** Snapshot manager. */
+ private IgniteCacheSnapshotManager snapshotMgr;
+
+ /** */
+ private PersistenceMetricsImpl persStoreMetrics;
+
+ /** */
+ private ObjectName persistenceMetricsMbeanName;
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public GridCacheDatabaseSharedManager(GridKernalContext ctx) {
+ IgniteConfiguration cfg = ctx.config();
+
+ persistenceCfg = cfg.getPersistentStoreConfiguration();
+
+ assert persistenceCfg != null : "PageStore should not be created if persistence is disabled.";
+
+ checkpointFreq = persistenceCfg.getCheckpointingFrequency();
+
+ lockWaitTime = persistenceCfg.getLockWaitTime();
+
+ final int pageSize = cfg.getMemoryConfiguration().getPageSize();
+
+ threadBuf = new ThreadLocal<ByteBuffer>() {
+ /** {@inheritDoc} */
+ @Override protected ByteBuffer initialValue() {
+ ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(pageSize);
+
+ tmpWriteBuf.order(ByteOrder.nativeOrder());
+
+ return tmpWriteBuf;
+ }
+ };
+
+ persStoreMetrics = new PersistenceMetricsImpl(
+ persistenceCfg.isMetricsEnabled(),
+ persistenceCfg.getRateTimeInterval(),
+ persistenceCfg.getSubIntervals()
+ );
+ }
+
+ /**
+ *
+ */
+ public Checkpointer getCheckpointer() {
+ return checkpointer;
+ }
+
+ /**
+ * For test use only.
+ */
+ public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
+ GridFutureAdapter<Void> fut = new GridFutureAdapter<>();
+
+ enableChangeApplied = fut;
+
+ checkpointsEnabled = enable;
+
+ wakeupForCheckpoint("enableCheckpoints()");
+
+ return fut;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void start0() throws IgniteCheckedException {
+ snapshotMgr = cctx.snapshot();
+
+ assert !cctx.kernalContext().state().active() : "Cluster with persistent must starting as inactive.";
+
+ if (!cctx.kernalContext().clientNode()) {
+ IgnitePageStoreManager store = cctx.pageStore();
+
+ assert store instanceof FilePageStoreManager : "Invalid page store manager was created: " + store;
+
+ storeMgr = (FilePageStoreManager)store;
+
+ cpDir = Paths.get(storeMgr.workDir().getAbsolutePath(), "cp").toFile();
+
+ if (!U.mkdirs(cpDir))
+ throw new IgniteCheckedException("Could not create directory for checkpoint metadata: " + cpDir);
+
+ fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log);
+
+ persStoreMetrics.wal(cctx.wal());
+
+ try {
+ persistenceMetricsMbeanName = U.registerMBean(
+ cctx.kernalContext().config().getMBeanServer(),
+ cctx.kernalContext().igniteInstanceName(),
+ "Persistent Store",
+ "PersistenceMetrics",
+ persStoreMetrics,
+ PersistenceMetricsMXBean.class);
+ }
+ catch (JMException e) {
+ throw new IgniteCheckedException("Failed to register persistence metrics MBean", e);
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ @Override public void initDataBase() throws IgniteCheckedException {
+ Long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize();
+
+ if (persistenceCfg.getCheckpointingThreads() > 1)
+ asyncRunner = new ThreadPoolExecutor(
+ persistenceCfg.getCheckpointingThreads(),
+ persistenceCfg.getCheckpointingThreads(),
+ 30L,
+ TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>()
+ );
+
+ // Intentionally use identity comparison to check if configuration default has changed.
+ //noinspection NumberEquality
+ if (cpBufSize == PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE) {
+ MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration();
+
+ assert memCfg != null;
+
+ long totalSize = memCfg.getSystemCacheMaxSize();
+
+ if (memCfg.getMemoryPolicies() == null)
+ totalSize += MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
+ else {
+ for (MemoryPolicyConfiguration memPlc : memCfg.getMemoryPolicies()) {
+ if (Long.MAX_VALUE - memPlc.getMaxSize() > totalSize)
+ totalSize += memPlc.getMaxSize();
+ else {
+ totalSize = Long.MAX_VALUE;
+
+ break;
+ }
+ }
+
+ assert totalSize > 0;
+ }
+
+ // Limit the checkpoint page buffer size by 2GB.
+ long dfltSize = 2 * 1024L * 1024L * 1024L;
+
+ long adjusted = Math.min(totalSize / 4, dfltSize);
+
+ if (cpBufSize < adjusted) {
+ U.quietAndInfo(log,
+ "Default checkpoint page buffer size is too small, setting to an adjusted value: "
+ + U.readableSize(adjusted, false)
+ );
+
+ cpBufSize = adjusted;
+ }
+ }
+
+ checkpointPageBufSize = cpBufSize;
+
+ super.start0();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void initPageMemoryDataStructures(MemoryConfiguration dbCfg) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException {
+ if (reconnect || cctx.kernalContext().clientNode() || !cctx.kernalContext().state().active())
+ return;
+
+ if (persistenceEnabled() && cctx.kernalContext().state().active())
+ initDataBase();
+
+ GridCacheProcessor cachePrc = cctx.kernalContext().cache();
+
+ // Todo before join local node.
+
+ Collection<String> cacheNames = new HashSet<>();
+
+ // TODO IGNITE-5075 group descriptors.
+ for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration()) {
+ if (CU.isSystemCache(ccfg.getName())) {
+ storeMgr.initializeForCache(
+ cctx.cache().cacheDescriptors().get(ccfg.getName()).groupDescriptor(), new StoredCacheData(ccfg));
+
+ cacheNames.add(ccfg.getName());
+ }
+ }
+
+ for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration())
+ if (!CU.isSystemCache(ccfg.getName())) {
+ DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptors().get(ccfg.getName());
+
+ if (cacheDesc != null)
+ storeMgr.initializeForCache(cacheDesc.groupDescriptor(), new StoredCacheData(ccfg));
+
+ cacheNames.add(ccfg.getName());
+ }
+
+ for (StoredCacheData cacheData : cctx.pageStore().readCacheConfigurations().values()) {
+ if (!cacheNames.contains(cacheData.config().getName()))
+ storeMgr.initializeForCache(
+ cctx.cache().cacheDescriptors().get(cacheData.config().getName()).groupDescriptor(), cacheData);
+ }
+
+ readCheckpointAndRestoreMemory();
+ }
+
+
+ /** {@inheritDoc} */
+ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ snapshotMgr = cctx.snapshot();
+
+ if (cctx.localNode().isClient())
+ return;
+
+ initDataBase();
+
+ if (log.isDebugEnabled())
+ log.debug("Activate database manager [id=" + cctx.localNodeId() +
+ " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
+
+ GridCacheProcessor cachePrc = cctx.kernalContext().cache();
+
+ // Todo join local info.
+
+ Collection<String> cacheNames = new HashSet<>();
+
+ // TODO IGNITE-5075 group descriptors.
+ for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration()) {
+ if (CU.isSystemCache(ccfg.getName())) {
+ storeMgr.initializeForCache(
+ cctx.cache().cacheDescriptors().get(ccfg.getName()).groupDescriptor(), new StoredCacheData(ccfg));
+
+ cacheNames.add(ccfg.getName());
+ }
+ }
+
+ for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration())
+ if (!CU.isSystemCache(ccfg.getName())) {
+ DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptors().get(ccfg.getName());
+
+ if (cacheDesc != null)
+ storeMgr.initializeForCache(cacheDesc.groupDescriptor(), new StoredCacheData(ccfg));
+
+ cacheNames.add(ccfg.getName());
+ }
+
+ for (StoredCacheData cacheData : cctx.pageStore().readCacheConfigurations().values()) {
+ if (!cacheNames.contains(cacheData.config().getName()))
+ storeMgr.initializeForCache(
+ cctx.cache().cacheDescriptors().get(cacheData.config().getName()).groupDescriptor(), cacheData);
+ }
+
+ readCheckpointAndRestoreMemory();
+
+ if (log.isDebugEnabled())
+ log.debug("Restore state after activation [nodeId=" + cctx.localNodeId() + " ]");
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ stop0(true);
+
+ if (log.isDebugEnabled())
+ log.debug("DeActivate database manager [id=" + cctx.localNodeId() +
+ " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
+
+ onKernalStop0(false);
+
+ /* Must be here, because after deactivate we can invoke activate and file lock must be already configured */
+ stopping = false;
+
+ fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log);
+ }
+
+ /**
+ *
+ */
+ private void readCheckpointAndRestoreMemory() throws IgniteCheckedException {
+ checkpointReadLock();
+
+ try {
+ CheckpointStatus status = readCheckpointStatus();
+
+ // First, bring memory to the last consistent checkpoint state if needed.
+ // This method should return a pointer to the last valid record in the WAL.
+ WALPointer restore = restoreMemory(status);
+
+ cctx.wal().resumeLogging(restore);
+
+ cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+ }
+ catch (StorageException e) {
+ throw new IgniteCheckedException(e);
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void lock() throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Try to capture file lock [nodeId=" +
+ cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
+
+ fileLockHolder.tryLock(lockWaitTime);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void unLock() {
+ if (log.isDebugEnabled())
+ log.debug("Release file lock [nodeId=" +
+ cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
+
+ fileLockHolder.release();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCacheStop(GridCacheContext cctx) {
+ snapshotMgr.onCacheStop(cctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onKernalStop0(boolean cancel) {
+ checkpointLock.writeLock().lock();
+
+ try {
+ stopping = true;
+ }
+ finally {
+ checkpointLock.writeLock().unlock();
+ }
+
+ snapshotMgr.onKernalStop(cancel);
+
+ shutdownCheckpointer(cancel);
+
+ lsnrs.clear();
+
+ super.onKernalStop0(cancel);
+
+ if (!cctx.kernalContext().clientNode()) {
+ unLock();
+
+ fileLockHolder.close();
+ }
+
+ if (persistenceMetricsMbeanName != null) {
+ try {
+ cctx.kernalContext().config().getMBeanServer().unregisterMBean(persistenceMetricsMbeanName);
+ }
+ catch (InstanceNotFoundException ignore) {
+ // No-op, nothing to unregister.
+ }
+ catch (MBeanRegistrationException e) {
+ U.error(log, "Failed to unregister persistence metrics MBean (will continue stop routine)", e);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void stop0(boolean cancel) {
+ super.stop0(cancel);
+
+ snapshotMgr.stop(cancel);
+ }
+
+ /** */
+ private long[] calculateFragmentSizes(int concLvl, long cacheSize) {
+ if (concLvl < 2)
+ concLvl = Runtime.getRuntime().availableProcessors();
+
+ long fragmentSize = cacheSize / concLvl;
+
+ if (fragmentSize < 1024 * 1024)
+ fragmentSize = 1024 * 1024;
+
+ long[] sizes = new long[concLvl + 1];
+
+ for (int i = 0; i < concLvl; i++)
+ sizes[i] = fragmentSize;
+
+ sizes[concLvl] = checkpointPageBufSize;
+
+ return sizes;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected PageMemory createPageMemory(
+ DirectMemoryProvider memProvider,
+ MemoryConfiguration memCfg,
+ MemoryPolicyConfiguration plcCfg,
+ MemoryMetricsImpl memMetrics
+ ) {
+ memMetrics.persistenceEnabled(true);
+
+ PageMemoryImpl pageMem = new PageMemoryImpl(
+ memProvider,
+ calculateFragmentSizes(
+ memCfg.getConcurrencyLevel(),
+ plcCfg.getMaxSize()
+ ),
+ cctx,
+ memCfg.getPageSize(),
+ new GridInClosure3X<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(
+ FullPageId fullId,
+ ByteBuffer pageBuf,
+ Integer tag
+ ) throws IgniteCheckedException {
+ storeMgr.write(fullId.cacheId(), fullId.pageId(), pageBuf, tag);
+
+ snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag);
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(
+ Long page,
+ FullPageId fullId,
+ PageMemoryEx pageMem
+ ) throws IgniteCheckedException {
+ snapshotMgr.onChangeTrackerPage(page, fullId, pageMem);
+ }
+ },
+ this,
+ memMetrics
+ );
+
+ memMetrics.pageMemory(pageMem);
+
+ return pageMem;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void checkPolicyEvictionProperties(MemoryPolicyConfiguration plcCfg, MemoryConfiguration dbCfg)
+ throws IgniteCheckedException {
+ if (plcCfg.getPageEvictionMode() != DataPageEvictionMode.DISABLED)
+ throw new IgniteCheckedException("Page eviction is not compatible with persistence: " + plcCfg.getName());
+ }
+
+ /**
+ * @param cancel Cancel flag.
+ */
+ @SuppressWarnings("unused")
+ private void shutdownCheckpointer(boolean cancel) {
+ Checkpointer cp = checkpointer;
+
+ if (cp != null) {
+ if (cancel)
+ cp.shutdownNow();
+ else
+ cp.cancel();
+
+ try {
+ U.join(cp);
+
+ checkpointer = null;
+ }
+ catch (IgniteInterruptedCheckedException ignore) {
+ U.warn(log, "Was interrupted while waiting for checkpointer shutdown, " +
+ "will not wait for checkpoint to finish.");
+
+ cp.shutdownNow();
+
+ while (true) {
+ try {
+ U.join(cp);
+
+ checkpointer = null;
+
+ cp.scheduledCp.cpFinishFut.onDone(
+ new NodeStoppingException("Checkpointer is stopped during node stop."));
+
+ break;
+ }
+ catch (IgniteInterruptedCheckedException ignored) {
+ //Ignore
+ }
+ }
+
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ if (asyncRunner != null) {
+ asyncRunner.shutdownNow();
+
+ try {
+ asyncRunner.awaitTermination(2, TimeUnit.MINUTES);
+ }
+ catch (InterruptedException ignore) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException {
+ DiscoveryEvent discoEvt = fut.discoveryEvent();
+
+ boolean joinEvt = discoEvt.type() == EventType.EVT_NODE_JOINED;
+
+ boolean locNode = discoEvt.eventNode().isLocal();
+
+ boolean isSrvNode = !cctx.kernalContext().clientNode();
+
+ boolean clusterStatusActive = cctx.kernalContext().state().active();
+
+ boolean clusterInTransitionStateToActive = fut.newClusterState() == ClusterState.ACTIVE;
+
+ // Before local node join event.
+ if (clusterInTransitionStateToActive ||
+ (joinEvt && locNode && isSrvNode && clusterStatusActive))
+ restoreState();
+
+ if (cctx.kernalContext().query().moduleEnabled()) {
+ for (GridCacheContext cacheCtx : (Collection<GridCacheContext>)cctx.cacheContexts()) {
+ if (cacheCtx.startTopologyVersion().equals(fut.topologyVersion()) &&
+ !cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode()) {
+ final int cacheId = cacheCtx.cacheId();
+
+ final IgniteInternalFuture<?> rebuildFut = cctx.kernalContext().query()
+ .rebuildIndexesFromHash(Collections.singletonList(cacheCtx.cacheId()));
+
+ idxRebuildFuts.put(cacheId, rebuildFut);
+
+ rebuildFut.listen(new CI1<IgniteInternalFuture>() {
+ @Override public void apply(IgniteInternalFuture igniteInternalFut) {
+ idxRebuildFuts.remove(cacheId, rebuildFut);
+ }
+ });
+ }
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgniteInternalFuture indexRebuildFuture(int cacheId) {
+ return idxRebuildFuts.get(cacheId);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean persistenceEnabled() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onCacheGroupsStopped(
+ Collection<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGrps) {
+ try {
+ waitForCheckpoint("caches stop");
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to wait for checkpoint finish during cache stop.", e);
+ }
+
+ Map<PageMemoryEx, Collection<Integer>> destroyed = new HashMap<>();
+
+ for (IgniteBiTuple<CacheGroupContext, Boolean> tup : stoppedGrps) {
+ PageMemoryEx pageMem = (PageMemoryEx)tup.get1().memoryPolicy().pageMemory();
+
+ Collection<Integer> grpIds = destroyed.get(pageMem);
+
+ if (grpIds == null) {
+ grpIds = new HashSet<>();
+
+ destroyed.put(pageMem, grpIds);
+ }
+
+ grpIds.add(tup.get1().groupId());
+
+ pageMem.onCacheGroupDestroyed(tup.get1().groupId());
+ }
+
+ Collection<IgniteInternalFuture<Void>> clearFuts = new ArrayList<>(destroyed.size());
+
+ for (Map.Entry<PageMemoryEx, Collection<Integer>> entry : destroyed.entrySet()) {
+ final Collection<Integer> grpIds = entry.getValue();
+
+ clearFuts.add(entry.getKey().clearAsync(new P3<Integer, Long, Integer>() {
+ @Override public boolean apply(Integer grpId, Long pageId, Integer tag) {
+ return grpIds.contains(grpId);
+ }
+ }, false));
+ }
+ for (IgniteInternalFuture<Void> clearFut : clearFuts) {
+ try {
+ clearFut.get();
+ }
+ catch (IgniteCheckedException e) {
+ log.error("Failed to clear page memory", e);
+ }
+ }
+
+ if (cctx.pageStore() != null) {
+ for (IgniteBiTuple<CacheGroupContext, Boolean> tup : stoppedGrps) {
+ CacheGroupContext grp = tup.get1();
+
+ if (grp.affinityNode()) {
+ try {
+ cctx.pageStore().shutdownForCacheGroup(grp, tup.get2());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to gracefully clean page store resources for destroyed cache " +
+ "[cache=" + grp.cacheOrGroupName() + "]", e);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Gets the checkpoint read lock. While this lock is held, checkpoint thread will not acquiSnapshotWorkerre memory state.
+ */
+ @SuppressWarnings("LockAcquiredButNotSafelyReleased")
+ @Override public void checkpointReadLock() {
+ if (checkpointLock.writeLock().isHeldByCurrentThread())
+ return;
+
+ for (; ; ) {
+ checkpointLock.readLock().lock();
+
+ if (stopping) {
+ checkpointLock.readLock().unlock();
+
+ throw new RuntimeException("Failed to perform cache update: node is stopping.");
+ }
+
+ if (safeToUpdatePageMemories() || checkpointLock.getReadHoldCount() > 1)
+ break;
+ else {
+ checkpointLock.readLock().unlock();
+
+ try {
+ checkpointer.wakeupForCheckpoint(0, "too many dirty pages").cpBeginFut.get();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException("Failed to wait for checkpoint begin.", e);
+ }
+ }
+ }
+
+ if (ASSERTION_ENABLED)
+ CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() + 1);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return !ASSERTION_ENABLED ||
+ checkpointLock.isWriteLockedByCurrentThread() ||
+ CHECKPOINT_LOCK_HOLD_COUNT.get() > 0;
+ }
+
+ /**
+ * @return {@code true} if all PageMemory instances are safe to update.
+ */
+ private boolean safeToUpdatePageMemories() {
+ Collection<MemoryPolicy> memPlcs = context().database().memoryPolicies();
+
+ if (memPlcs == null)
+ return true;
+
+ for (MemoryPolicy memPlc : memPlcs) {
+ PageMemoryEx pageMemEx = (PageMemoryEx) memPlc.pageMemory();
+
+ if (!pageMemEx.safeToUpdate())
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Releases the checkpoint read lock.
+ */
+ @Override public void checkpointReadUnlock() {
+ if (checkpointLock.writeLock().isHeldByCurrentThread())
+ return;
+
+ checkpointLock.readLock().unlock();
+
+ if (checkpointer != null) {
+ Collection<MemoryPolicy> memPlcs = context().database().memoryPolicies();
+
+ if (memPlcs != null) {
+ for (MemoryPolicy memPlc : memPlcs) {
+ PageMemoryEx mem = (PageMemoryEx)memPlc.pageMemory();
+
+ if (mem != null && !mem.safeToUpdate()) {
+ checkpointer.wakeupForCheckpoint(0, "too many dirty pages");
+
+ break;
+ }
+ }
+ }
+ }
+
+ if (ASSERTION_ENABLED)
+ CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() - 1);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed to restore database status from WAL.
+ */
+ private void restoreState() throws IgniteCheckedException {
+ try {
+ CheckpointStatus status = readCheckpointStatus();
+
+ checkpointReadLock();
+
+ try {
+ applyLastUpdates(status);
+ }
+ finally {
+ checkpointReadUnlock();
+ }
+
+ snapshotMgr.restoreState();
+
+ checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
+
+ new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
+ }
+ catch (StorageException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
+ assert reservedForExchange == null : reservedForExchange;
+
+ reservedForExchange = new HashMap<>();
+
+ for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+ if (grp.isLocal())
+ continue;
+
+ for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
+ if (part.state() != GridDhtPartitionState.OWNING || part.dataStore().fullSize() <= walRebalanceThreshold)
+ continue;
+
+ CheckpointEntry cpEntry = searchCheckpointEntry(grp.groupId(), part.id(), null);
+
+ try {
+ if (cpEntry != null && cctx.wal().reserve(cpEntry.cpMark)) {
+ Map<Integer, T2<Long, WALPointer>> cacheMap = reservedForExchange.get(grp.groupId());
+
+ if (cacheMap == null) {
+ cacheMap = new HashMap<>();
+
+ reservedForExchange.put(grp.groupId(), cacheMap);
+ }
+
+ cacheMap.put(part.id(), new T2<>(cpEntry.partitionCounter(grp.groupId(), part.id()), cpEntry.cpMark));
+ }
+ }
+ catch (IgniteCheckedException ex) {
+ U.error(log, "Error while trying to reserve history", ex);
+ }
+ }
+ }
+
+ Map<Integer, Map<Integer, Long>> resMap = new HashMap<>();
+
+ for (Map.Entry<Integer, Map<Integer, T2<Long, WALPointer>>> e : reservedForExchange.entrySet()) {
+ Map<Integer, Long> cacheMap = new HashMap<>();
+
+ for (Map.Entry<Integer, T2<Long, WALPointer>> e0 : e.getValue().entrySet())
+ cacheMap.put(e0.getKey(), e0.getValue().get1());
+
+ resMap.put(e.getKey(), cacheMap);
+ }
+
+ return resMap;
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void releaseHistoryForExchange() {
+ if (reservedForExchange == null)
+ return;
+
+ for (Map.Entry<Integer, Map<Integer, T2<Long, WALPointer>>> e : reservedForExchange.entrySet()) {
+ for (Map.Entry<Integer, T2<Long, WALPointer>> e0 : e.getValue().entrySet()) {
+ try {
+ cctx.wal().release(e0.getValue().get2());
+ }
+ catch (IgniteCheckedException ex) {
+ U.error(log, "Could not release history lock", ex);
+ }
+ }
+ }
+
+ reservedForExchange = null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserveHistoryForPreloading(int grpId, int partId, long cntr) {
+ CheckpointEntry cpEntry = searchCheckpointEntry(grpId, partId, cntr);
+
+ if (cpEntry == null)
+ return false;
+
+ WALPointer ptr = cpEntry.cpMark;
+
+ if (ptr == null)
+ return false;
+
+ boolean reserved;
+
+ try {
+ reserved = cctx.wal().reserve(ptr);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Error while trying to reserve history", e);
+
+ reserved = false;
+ }
+
+ if (reserved)
+ reservedForPreloading.put(new T2<>(grpId, partId), new T2<>(cntr, ptr));
+
+ return reserved;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void releaseHistoryForPreloading() {
+ for (Map.Entry<T2<Integer, Integer>, T2<Long, WALPointer>> e : reservedForPreloading.entrySet()) {
+ try {
+ cctx.wal().release(e.getValue().get2());
+ }
+ catch (IgniteCheckedException ex) {
+ U.error(log, "Could not release WAL reservation", ex);
+
+ throw new IgniteException(ex);
+ }
+ }
+
+ reservedForPreloading.clear();
+ }
+
+ /**
+ * For debugging only. TODO: remove.
+ *
+ */
+ public Map<T2<Integer, Integer>, T2<Long, WALPointer>> reservedForPreloading() {
+ return reservedForPreloading;
+ }
+
+ /**
+ *
+ */
+ @Nullable @Override public IgniteInternalFuture wakeupForCheckpoint(String reason) {
+ Checkpointer cp = checkpointer;
+
+ if (cp != null)
+ return cp.wakeupForCheckpoint(0, reason).cpBeginFut;
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void waitForCheckpoint(String reason) throws IgniteCheckedException {
+ Checkpointer cp = checkpointer;
+
+ if (cp == null)
+ return;
+
+ CheckpointProgressSnapshot progSnapshot = cp.wakeupForCheckpoint(0, reason);
+
+ IgniteInternalFuture fut1 = progSnapshot.cpFinishFut;
+
+ fut1.get();
+
+ if (!progSnapshot.started)
+ return;
+
+ IgniteInternalFuture fut2 = cp.wakeupForCheckpoint(0, reason).cpFinishFut;
+
+ assert fut1 != fut2;
+
+ fut2.get();
+ }
+
+ /**
+ * Tries to search for a WAL pointer for the given partition counter start.
+ *
+ * @param grpId Cache group ID.
+ * @param part Partition ID.
+ * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
+ * @return Checkpoint entry or {@code null} if failed to search.
+ */
+ @Nullable public WALPointer searchPartitionCounter(int grpId, int part, @Nullable Long partCntrSince) {
+ CheckpointEntry entry = searchCheckpointEntry(grpId, part, partCntrSince);
+
+ if (entry == null)
+ return null;
+
+ return entry.cpMark;
+ }
+
+ /**
+ * Tries to search for a WAL pointer for the given partition counter start.
+ *
+ * @param grpId Cache group ID.
+ * @param part Partition ID.
+ * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
+ * @return Checkpoint entry or {@code null} if failed to search.
+ */
+ @Nullable private CheckpointEntry searchCheckpointEntry(int grpId, int part, @Nullable Long partCntrSince) {
+ boolean hasGap = false;
+ CheckpointEntry first = null;
+
+ for (Long cpTs : checkpointHist.checkpoints()) {
+ try {
+ CheckpointEntry entry = checkpointHist.entry(cpTs);
+
+ Long foundCntr = entry.partitionCounter(grpId, part);
+
+ if (foundCntr != null) {
+ if (partCntrSince == null) {
+ if (hasGap) {
+ first = entry;
+
+ hasGap = false;
+ }
+
+ if (first == null)
+ first = entry;
+ }
+ else if (foundCntr <= partCntrSince) {
+ first = entry;
+
+ hasGap = false;
+ }
+ else
+ return hasGap ? null : first;
+ }
+ else
+ hasGap = true;
+ }
+ catch (IgniteCheckedException ignore) {
+ // Treat exception the same way as a gap.
+ hasGap = true;
+ }
+ }
+
+ return hasGap ? null : first;
+ }
+
+ /**
+ * @return Checkpoint history. For tests only.
+ */
+ public CheckpointHistory checkpointHistory() {
+ return checkpointHist;
+ }
+
+ /**
+ * @return Checkpoint directory.
+ */
+ public File checkpointDirectory() {
+ return cpDir;
+ }
+
+ /**
+ * @param lsnr Listener.
+ */
+ public void addCheckpointListener(DbCheckpointListener lsnr) {
+ lsnrs.add(lsnr);
+ }
+
+ /**
+ * @param lsnr Listener.
+ */
+ public void removeCheckpointListener(DbCheckpointListener lsnr) {
+ lsnrs.remove(lsnr);
+ }
+
+ /**
+ * @return Read checkpoint status.
+ * @throws IgniteCheckedException If failed to read checkpoint status page.
+ */
+ @SuppressWarnings("TooBroadScope")
+ private CheckpointStatus readCheckpointStatus() throws IgniteCheckedException {
+ long lastStartTs = 0;
+ long lastEndTs = 0;
+
+ UUID startId = CheckpointStatus.NULL_UUID;
+ UUID endId = CheckpointStatus.NULL_UUID;
+
+ File startFile = null;
+ File endFile = null;
+
+ WALPointer startPtr = CheckpointStatus.NULL_PTR;
+ WALPointer endPtr = CheckpointStatus.NULL_PTR;
+
+ File dir = cpDir;
+
+ if (!dir.exists()) {
+ // TODO: remove excessive logging after GG-12116 fix.
+ File[] files = dir.listFiles();
+
+ if (files != null && files.length > 0) {
+ log.warning("Read checkpoint status: cpDir.exists() is false, cpDir.listFiles() is: " +
+ Arrays.toString(files));
+ }
+
+ if (Files.exists(dir.toPath()))
+ log.warning("Read checkpoint status: cpDir.exists() is false, Files.exists(cpDir) is true.");
+
+ log.info("Read checkpoint status: checkpoint directory is not found.");
+
+ return new CheckpointStatus(0, startId, startPtr, endId, endPtr);
+ }
+
+ File[] files = dir.listFiles();
+
+ for (File file : files) {
+ Matcher matcher = CP_FILE_NAME_PATTERN.matcher(file.getName());
+
+ if (matcher.matches()) {
+ long ts = Long.parseLong(matcher.group(1));
+ UUID id = UUID.fromString(matcher.group(2));
+ CheckpointEntryType type = CheckpointEntryType.valueOf(matcher.group(3));
+
+ if (type == CheckpointEntryType.START && ts > lastStartTs) {
+ lastStartTs = ts;
+ startId = id;
+ startFile = file;
+ }
+ else if (type == CheckpointEntryType.END && ts > lastEndTs) {
+ lastEndTs = ts;
+ endId = id;
+ endFile = file;
+ }
+ }
+ }
+
+ ByteBuffer buf = ByteBuffer.allocate(20);
+ buf.order(ByteOrder.nativeOrder());
+
+ if (startFile != null)
+ startPtr = readPointer(startFile, buf);
+
+ if (endFile != null)
+ endPtr = readPointer(endFile, buf);
+
+ // TODO: remove excessive logging after GG-12116 fix.
+ log.info("Read checkpoint status: start marker = " + startFile + ", end marker = " + endFile);
+
+ return new CheckpointStatus(lastStartTs, startId, startPtr, endId, endPtr);
+ }
+
+ /**
+ * @param cpMarkerFile Checkpoint mark file.
+ * @return WAL pointer.
+ * @throws IgniteCheckedException If failed to read mark file.
+ */
+ private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteCheckedException {
+ buf.position(0);
+
+ try (FileChannel ch = FileChannel.open(cpMarkerFile.toPath(), StandardOpenOption.READ)) {
+ ch.read(buf);
+
+ buf.flip();
+
+ return new FileWALPointer(buf.getLong(), buf.getInt(), buf.getInt());
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to read checkpoint pointer from marker file: " +
+ cpMarkerFile.getAbsolutePath(), e);
+ }
+ }
+
+ /**
+ * @param status Checkpoint status.
+ */
+ private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
+ if (log.isInfoEnabled())
+ log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked="
+ + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
+
+ boolean apply = status.needRestoreMemory();
+
+ if (apply) {
+ U.quietAndWarn(log, "Ignite node crashed in the middle of checkpoint. Will restore memory state and " +
+ "enforce checkpoint on node start.");
+
+ cctx.pageStore().beginRecover();
+ }
+
+ long start = U.currentTimeMillis();
+ int applied = 0;
+ WALPointer lastRead = null;
+
+ try (WALIterator it = cctx.wal().replay(status.endPtr)) {
+ while (it.hasNextX()) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+ WALRecord rec = tup.get2();
+
+ lastRead = tup.get1();
+
+ switch (rec.type()) {
+ case CHECKPOINT_RECORD:
+ CheckpointRecord cpRec = (CheckpointRecord)rec;
+
+ // We roll memory up until we find a checkpoint start record registered in the status.
+ if (F.eq(cpRec.checkpointId(), status.cpStartId)) {
+ log.info("Found last checkpoint marker [cpId=" + cpRec.checkpointId() +
+ ", pos=" + tup.get1() + ']');
+
+ apply = false;
+ }
+ else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
+ U.warn(log, "Found unexpected checkpoint marker, skipping [cpId=" + cpRec.checkpointId() +
+ ", expCpId=" + status.cpStartId + ", pos=" + tup.get1() + ']');
+
+ break;
+
+ case PAGE_RECORD:
+ if (apply) {
+ PageSnapshot pageRec = (PageSnapshot)rec;
+
+ // Here we do not require tag check because we may be applying memory changes after
+ // several repetitive restarts and the same pages may have changed several times.
+ int cacheId = pageRec.fullPageId().cacheId();
+ long pageId = pageRec.fullPageId().pageId();
+
+ PageMemoryEx pageMem = getPageMemoryForCacheGroup(cacheId);
+
+ long page = pageMem.acquirePage(cacheId, pageId, true);
+
+ try {
+ long pageAddr = pageMem.writeLock(cacheId, pageId, page);
+
+ try {
+ PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
+ }
+ finally {
+ pageMem.writeUnlock(cacheId, pageId, page, null, true, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(cacheId, pageId, page);
+ }
+
+ applied++;
+ }
+
+ break;
+
+ case PARTITION_DESTROY:
+ if (apply) {
+ PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
+
+ final int cId = destroyRec.cacheId();
+ final int pId = destroyRec.partitionId();
+
+ PageMemoryEx pageMem = getPageMemoryForCacheGroup(cId);
+
+ pageMem.clearAsync(new P3<Integer, Long, Integer>() {
+ @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
+ return cacheId == cId && PageIdUtils.partId(pageId) == pId;
+ }
+ }, true).get();
+ }
+
+ break;
+
+ default:
+ if (apply && rec instanceof PageDeltaRecord) {
+ PageDeltaRecord r = (PageDeltaRecord)rec;
+
+ int cacheId = r.cacheId();
+ long pageId = r.pageId();
+
+ PageMemoryEx pageMem = getPageMemoryForCacheGroup(cacheId);
+
+ // Here we do not require tag check because we may be applying memory changes after
+ // several repetitive restarts and the same pages may have changed several times.
+ long page = pageMem.acquirePage(cacheId, pageId, true);
+
+ try {
+ long pageAddr = pageMem.writeLock(cacheId, pageId, page);
+
+ try {
+ r.applyDelta(pageMem, pageAddr);
+ }
+ finally {
+ pageMem.writeUnlock(cacheId, pageId, page, null, true, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(cacheId, pageId, page);
+ }
+
+ applied++;
+ }
+ }
+ }
+ }
+
+ if (status.needRestoreMemory()) {
+ if (apply)
+ throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " +
+ "on disk, but checkpoint record is missed in WAL) " +
+ "[cpStatus=" + status + ", lastRead=" + lastRead + "]");
+
+ log.info("Finished applying memory changes [changesApplied=" + applied +
+ ", time=" + (U.currentTimeMillis() - start) + "ms]");
+
+ if (applied > 0)
+ finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr);
+ }
+
+ checkpointHist.loadHistory(cpDir);
+
+ return lastRead == null ? null : lastRead.next();
+ }
+
+ /**
+ * Obtains PageMemory reference from cache descriptor instead of cache context.
+ *
+ * @param grpId Cache group id.
+ * @return PageMemoryEx instance.
+ * @throws IgniteCheckedException if no MemoryPolicy is configured for a name obtained from cache descriptor.
+ */
+ private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedException {
+ // TODO IGNITE-5075: cache descriptor can be removed.
+ GridCacheSharedContext sharedCtx = context();
+
+ String memPlcName = sharedCtx
+ .cache()
+ .cacheGroupDescriptors().get(grpId)
+ .config()
+ .getMemoryPolicyName();
+
+ return (PageMemoryEx)sharedCtx.database().memoryPolicy(memPlcName).pageMemory();
+ }
+
+ /**
+ * @param status Last registered checkpoint status.
+ * @throws IgniteCheckedException If failed to apply updates.
+ * @throws StorageException If IO exception occurred while reading write-ahead log.
+ */
+ private void applyLastUpdates(CheckpointStatus status) throws IgniteCheckedException {
+ if (log.isInfoEnabled())
+ log.info("Applying lost cache updates since last checkpoint record [lastMarked="
+ + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
+
+ cctx.kernalContext().query().skipFieldLookup(true);
+
+ long start = U.currentTimeMillis();
+ int applied = 0;
+
+ try (WALIterator it = cctx.wal().replay(status.startPtr)) {
+ Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
+
+ while (it.hasNextX()) {
+ IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
+
+ WALRecord rec = next.get2();
+
+ switch (rec.type()) {
+ case DATA_RECORD:
+ DataRecord dataRec = (DataRecord)rec;
+
+ for (DataEntry dataEntry : dataRec.writeEntries()) {
+ int cacheId = dataEntry.cacheId();
+
+ GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
+
+ applyUpdate(cacheCtx, dataEntry);
+
+ applied++;
+ }
+
+ break;
+
+ case PART_META_UPDATE_STATE:
+ PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
+
+ partStates.put(new T2<>(metaStateRecord.cacheId(), metaStateRecord.partitionId()),
+ new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter()));
+
+ break;
+
+ default:
+ // Skip other records.
+ }
+ }
+
+ restorePartitionState(partStates);
+ }
+ finally {
+ cctx.kernalContext().query().skipFieldLookup(false);
+ }
+
+ if (log.isInfoEnabled())
+ log.info("Finished applying WAL changes [updatesApplied=" + applied +
+ ", time=" + (U.currentTimeMillis() - start) + "ms]");
+ }
+
+ /**
+ * @param partStates Partition states.
+ * @throws IgniteCheckedException If failed to restore.
+ */
+ private void restorePartitionState(
+ Map<T2<Integer, Integer>, T2<Integer, Long>> partStates
+ ) throws IgniteCheckedException {
+ for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+ int grpId = grp.groupId();
+
+ PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+
+ for (int i = 0; i < grp.affinity().partitions(); i++) {
+ if (storeMgr.exists(grpId, i)) {
+ storeMgr.ensure(grpId, i);
+
+ if (storeMgr.pages(grpId, i) <= 1)
+ continue;
+
+ long partMetaId = pageMem.partitionMetaPageId(grpId, i);
+ long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
+ try {
+ long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
+
+ boolean changed = false;
+
+ try {
+ PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
+
+ T2<Integer, Long> fromWal = partStates.get(new T2<>(grpId, i));
+
+ GridDhtLocalPartition part = grp.topology()
+ .localPartition(i, AffinityTopologyVersion.NONE, true);
+
+ assert part != null;
+
+ if (fromWal != null) {
+ int stateId = fromWal.get1();
+
+ io.setPartitionState(pageAddr, (byte)stateId);
+
+ changed = updateState(part, stateId);
+
+ if (stateId == GridDhtPartitionState.OWNING.ordinal()) {
+ grp.offheap().onPartitionInitialCounterUpdated(i, fromWal.get2());
+
+ if (part.initialUpdateCounter() < fromWal.get2()) {
+ part.initialUpdateCounter(fromWal.get2());
+
+ changed = true;
+ }
+ }
+ }
+ else
+ changed = updateState(part, (int)io.getPartitionState(pageAddr));
+ }
+ finally {
+ pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
+ }
+ }
+ finally {
+ pageMem.releasePage(grpId, partMetaId, partMetaPage);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * @param part Partition to restore state for.
+ * @param stateId State enum ordinal.
+ * @return Updated flag.
+ */
+ private boolean updateState(GridDhtLocalPartition part, int stateId) {
+ if (stateId != -1) {
+ GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId);
+
+ assert state != null;
+
+ part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state);
+
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * @param cacheCtx Cache context to apply an update.
+ * @param dataEntry Data entry to apply.
+ */
+ private void applyUpdate(GridCacheContext cacheCtx, DataEntry dataEntry) throws IgniteCheckedException {
+ GridDhtLocalPartition locPart = cacheCtx.topology()
+ .localPartition(dataEntry.partitionId(), AffinityTopologyVersion.NONE, true);
+
+ switch (dataEntry.op()) {
+ case CREATE:
+ case UPDATE:
+ cacheCtx.offheap().update(
+ cacheCtx,
+ dataEntry.key(),
+ dataEntry.value(),
+ dataEntry.writeVersion(),
+ 0L,
+ locPart,
+ null);
+
+ if (dataEntry.partitionCounter() != 0)
+ cacheCtx.offheap().onPartitionInitialCounterUpdated(dataEntry.partitionId(), dataEntry.partitionCounter());
+
+ break;
+
+ case DELETE:
+ cacheCtx.offheap().remove(cacheCtx, dataEntry.key(), dataEntry.partitionId(), locPart);
+
+ if (dataEntry.partitionCounter() != 0)
+ cacheCtx.offheap().onPartitionInitialCounterUpdated(dataEntry.partitionId(), dataEntry.partitionCounter());
+
+ break;
+
+ default:
+ throw new IgniteCheckedException("Invalid operation for WAL entry update: " + dataEntry.op());
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPtr) throws IgniteCheckedException {
+ assert cpTs != 0;
+
+ ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(pageSize());
+
+ long start = System.currentTimeMillis();
+
+ Collection<MemoryPolicy> memPolicies = context().database().memoryPolicies();
+
+ List<IgniteBiTuple<PageMemory, Collection<FullPageId>>> cpEntities = new ArrayList<>(memPolicies.size());
+
+ for (MemoryPolicy memPlc : memPolicies) {
+ PageMemoryEx pageMem = (PageMemoryEx) memPlc.pageMemory();
+ cpEntities.add(new IgniteBiTuple<PageMemory, Collection<FullPageId>>(pageMem,
+ (pageMem).beginCheckpoint()));
+ }
+
+ tmpWriteBuf.order(ByteOrder.nativeOrder());
+
+ // Identity stores set.
+ Collection<PageStore> updStores = new HashSet<>();
+
+ int cpPagesCnt = 0;
+
+ for (IgniteBiTuple<PageMemory, Collection<FullPageId>> e : cpEntities) {
+ PageMemoryEx pageMem = (PageMemoryEx) e.get1();
+
+ Collection<FullPageId> cpPages = e.get2();
+
+ cpPagesCnt += cpPages.size();
+
+ for (FullPageId fullId : cpPages) {
+ tmpWriteBuf.rewind();
+
+ Integer tag = pageMem.getForCheckpoint(fullId, tmpWriteBuf, null);
+
+ if (tag != null) {
+ tmpWriteBuf.rewind();
+
+ PageStore store = storeMgr.writeInternal(fullId.cacheId(), fullId.pageId(), tmpWriteBuf, tag);
+
+ tmpWriteBuf.rewind();
+
+ updStores.add(store);
+ }
+ }
+ }
+
+ long written = U.currentTimeMillis();
+
+ for (PageStore updStore : updStores)
+ updStore.sync();
+
+ long fsync = U.currentTimeMillis();
+
+ for (IgniteBiTuple<PageMemory, Collection<FullPageId>> e : cpEntities)
+ ((PageMemoryEx)e.get1()).finishCheckpoint();
+
+ writeCheckpointEntry(
+ tmpWriteBuf,
+ cpTs,
+ cpId,
+ walPtr,
+ null,
+ CheckpointEntryType.END);
+
+ cctx.pageStore().finishRecover();
+
+ if (log.isInfoEnabled())
+ log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
+ "pagesWrite=%dms, fsync=%dms, total=%dms]",
+ cpId,
+ cpPagesCnt,
+ walPtr,
+ written - start,
+ fsync - written,
+ fsync - start));
+ }
+
+ /**
+ * @param cpId Checkpoint ID.
+ * @param ptr Wal pointer of current checkpoint.
+ */
+ private CheckpointEntry writeCheckpointEntry(
+ ByteBuffer tmpWriteBuf,
+ long cpTs,
+ UUID cpId,
+ WALPointer ptr,
+ CheckpointRecord rec,
+ CheckpointEntryType type
+ ) throws IgniteCheckedException {
+ assert ptr instanceof FileWALPointer;
+
+ FileWALPointer filePtr = (FileWALPointer)ptr;
+
+ String fileName = checkpointFileName(cpTs, cpId, type);
+
+ try (FileChannel ch = FileChannel.open(Paths.get(cpDir.getAbsolutePath(), fileName),
+ StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND)) {
+
+ tmpWriteBuf.rewind();
+
+ tmpWriteBuf.putLong(filePtr.index());
+
+ tmpWriteBuf.putInt(filePtr.fileOffset());
+
+ tmpWriteBuf.putInt(filePtr.length());
+
+ tmpWriteBuf.flip();
+
+ ch.write(tmpWriteBuf);
+
+ tmpWriteBuf.clear();
+
+ if (!skipSync)
+ ch.force(true);
+
+ return type == CheckpointEntryType.START ?
+ new CheckpointEntry(cpTs, ptr, cpId, rec.cacheGroupStates()) : null;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /**
+ * @param cpTs Checkpoint timestamp.
+ * @param cpId Checkpoint ID.
+ * @param type Checkpoint type.
+ * @return Checkpoint file name.
+ */
+ private static String checkpointFileName(long cpTs, UUID cpId, CheckpointEntryType type) {
+ return cpTs + "-" + cpId + "-" + type + ".bin";
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("NakedNotify")
+ public class Checkpointer extends GridWorker {
+ /** Temporary write buffer. */
+ private final ByteBuffer tmpWriteBuf;
+
+ /** Next scheduled checkpoint progress. */
+ private volatile CheckpointProgress scheduledCp;
+
+ /** Current checkpoint. This field is updated only by checkpoint thread. */
+ private volatile CheckpointProgress curCpProgress;
+
+ /** Shutdown now. */
+ private volatile boolean shutdownNow;
+
+ /**
+ * @param gridName Grid name.
+ * @param name Thread name.
+ * @param log Logger.
+ */
+ protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log) {
+ super(gridName, name, log);
+
+ scheduledCp = new CheckpointProgress(U.currentTimeMillis() + checkpointFreq);
+
+ tmpWriteBuf = ByteBuffer.allocateDirect(pageSize());
+
+ tmpWriteBuf.order(ByteOrder.nativeOrder());
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ while (!isCancelled()) {
+ waitCheckpointEvent();
+
+ GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
+
+ if (enableChangeApplied != null) {
+ enableChangeApplied.onDone();
+
+ GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
+ }
+
+ if (checkpointsEnabled)
+ doCheckpoint();
+ else {
+ synchronized (this) {
+ scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
+ }
+ }
+ }
+
+ // Final run after the cancellation.
+ if (checkpointsEnabled && !shutdownNow)
+ doCheckpoint();
+
+ scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+ }
+
+ /**
+ *
+ */
+ private CheckpointProgressSnapshot wakeupForCheckpoint(long delayFromNow, String reason) {
+ CheckpointProgress sched = scheduledCp;
+
+ long next = U.currentTimeMillis() + delayFromNow;
+
+ if (sched.nextCpTs <= next)
+ return new CheckpointProgressSnapshot(sched);
+
+ CheckpointProgressSnapshot ret;
+
+ synchronized (this) {
+ sched = scheduledCp;
+
+ if (sched.nextCpTs > next) {
+ sched.reason = reason;
+
+ sched.nextCpTs = next;
+ }
+
+ ret = new CheckpointProgressSnapshot(sched);
+
+ notifyAll();
+ }
+
+ return ret;
+ }
+
+ /**
+ * @param snapshotOperation Snapshot operation.
+ */
+ public IgniteInternalFuture wakeupForSnapshotCreation(SnapshotOperation snapshotOperation) {
+ GridFutureAdapter<Object> ret;
+
+ synchronized (this) {
+ scheduledCp.nextCpTs = U.currentTimeMillis();
+
+ scheduledCp.reason = "snapshot";
+
+ scheduledCp.nextSnapshot = true;
+
+ scheduledCp.snapshotOperation = snapshotOperation;
+
+ ret = scheduledCp.cpBeginFut;
+
+ notifyAll();
+ }
+
+ return ret;
+ }
+
+ /**
+ *
+ */
+ private void doCheckpoint() {
+ try {
+ CheckpointMetricsTracker tracker = new CheckpointMetricsTracker();
+
+ Checkpoint chp = markCheckpointBegin(tracker);
+
+ if (chp.cpPages == null)
+ return;
+
+ snapshotMgr.onCheckPointBegin();
+
+ boolean interrupted = true;
+
+ try {
+ if (chp.hasDelta()) {
+ // Identity stores set.
+ GridConcurrentHashSet<PageStore> updStores = new GridConcurrentHashSet<>();
+
+ CountDownFuture doneWriteFut = new CountDownFuture(
+ asyncRunner == null ? 1 : chp.cpPages.collectionsSize());
+
+ tracker.onPagesWriteStart();
+
+ if (asyncRunner != null) {
+ for (int i = 0; i < chp.cpPages.collectionsSize(); i++) {
+ Runnable write = new WriteCheckpointPages(
+ tracker,
+ chp.cpPages.innerCollection(i),
+ updStores,
+ doneWriteFut
+ );
+
+ try {
+ asyncRunner.execute(write);
+ }
+ catch (RejectedExecutionException ignore) {
+ // Run the task synchronously.
+ write.run();
+ }
+ }
+ }
+ else {
+ // Single-threaded checkpoint.
+ Runnable write = new WriteCheckpointPages(tracker, chp.cpPages, updStores, doneWriteFut);
+
+ write.run();
+ }
+
+ // Wait and check for errors.
+ doneWriteFut.get();
+
+ // Must re-check shutdown flag here because threads may have skipped some pages.
+ // If so, we should not put finish checkpoint mark.
+ if (shutdownNow) {
+ chp.progress.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+
+ return;
+ }
+
+ snapshotMgr.afterCheckpointPageWritten();
+
+ tracker.onFsyncStart();
+
+ if (!skipSync) {
+ for (PageStore updStore : updStores) {
+ if (shutdownNow) {
+ chp.progress.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
+
+ return;
+ }
+
+ updStore.sync();
+ }
+ }
+ }
+ else {
+ tracker.onPagesWriteStart();
+ tracker.onFsyncStart();
+ }
+
+ // Must mark successful checkpoint only if there are no exceptions or interrupts.
+ interrupted = false;
+ }
+ finally {
+ if (!interrupted)
+ markCheckpointEnd(chp);
+ }
+
+ tracker.onEnd();
+
+ if (chp.hasDelta()) {
+ if (printCheckpointStats) {
+ if (log.isInfoEnabled())
+ log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
+ "walSegmentsCleared=%d, markDuration=%dms, pagesWrite=%dms, fsync=%dms, " +
+ "total=%dms]",
+ chp.cpEntry.checkpointId(),
+ chp.pagesSize,
+ chp.cpEntry.checkpointMark(),
+ chp.walFilesDeleted,
+ tracker.markDuration(),
+ tracker.pagesWriteDuration(),
+ tracker.fsyncDuration(),
+ tracker.totalDuration()));
+ }
+
+ persStoreMetrics.onCheckpoint(
+ tracker.lockWaitDuration(),
+ tracker.markDuration(),
+ tracker.pagesWriteDuration(),
+ tracker.fsyncDuration(),
+ tracker.totalDuration(),
+ chp.pagesSize,
+ tracker.dataPagesWritten(),
+ tracker.cowPagesWritten());
+ }
+ else {
+ persStoreMetrics.onCheckpoint(
+ tracker.lockWaitDuration(),
+ tracker.markDuration(),
+ tracker.pagesWriteDuration(),
+ tracker.fsyncDuration(),
+ tracker.totalDuration(),
+ chp.pagesSize,
+ tracker.dataPagesWritten(),
+ tracker.cowPagesWritten());
+ }
+ }
+ catch (IgniteCheckedException e) {
+ // TODO-ignite-db how to handle exception?
+ U.error(log, "Failed to create checkpoint.", e);
+ }
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("WaitNotInLoop")
+ private void waitCheckpointEvent() {
+ boolean cancel = false;
+
+ try {
+ long now = U.currentTimeMillis();
+
+ synchronized (this) {
+ long remaining;
+
+ while ((remaining = scheduledCp.nextCpTs - now) > 0 && !isCancelled()) {
+ wait(remaining);
+
+ now = U.currentTimeMillis();
+ }
+ }
+ }
+ catch (InterruptedException ignored) {
+ Thread.currentThread().interrupt();
+
+ cancel = true;
+ }
+
+ if (cancel)
+ isCancelled = true;
+ }
+
+ /**
+ *
+ */
+ @SuppressWarnings("TooBroadScope")
+ private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws IgniteCheckedException {
+ CheckpointRecord cpRec = new CheckpointRecord(null, false);
+
+ WALPointer cpPtr = null;
+
+ GridMultiCollectionWrapper<FullPageId> cpPages;
+
+ final CheckpointProgress curr;
+
+ tracker.onLockWaitStart();
+
+ checkpointLock.writeLock().lock();
+
+ try {
+ tracker.onMarkStart();
+
+ synchronized (this) {
+ curr = scheduledCp;
+
+ curr.started = true;
+
+ if (curr.reason == null)
+ curr.reason = "timeout";
+
+ // It is important that we assign a new progress object before checkpoint mark in page memory.
+ scheduledCp = new CheckpointProgress(U.currentTimeMillis() + checkpointFreq);
+
+ curCpProgress = curr;
+ }
+
+ final NavigableMap<T2<Integer, Integer>, T2<Integer, Integer>> map =
+ new TreeMap<>(FullPageIdIterableComparator.INSTANCE);
+
+ DbCheckpointListener.Context ctx0 = new DbCheckpointListener.Context() {
+ @Override public boolean nextSnapshot() {
+ return curr.nextSnapshot;
+ }
+
+ @Override public Map<T2<Integer, Integer>, T2<Integer, Integer>> partitionStatMap() {
+ return map;
+ }
+ };
+
+ // Listeners must be invoked before we write checkpoint record to WAL.
+ for (DbCheckpointListener lsnr : lsnrs)
+ lsnr.onCheckpointBegin(ctx0);
+
+ for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
+ if (grp.isLocal())
+ continue;
+
+ List<GridDhtLocalPartition> locParts = new ArrayList<>();
+
+ for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
+ locParts.add(part);
+
+ Collections.sort(locParts, ASC_PART_COMPARATOR);
+
+ CacheState state = new CacheState(locParts.size());
+
+ for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
+ state.addPartitionState(part.id(), part.dataStore().fullSize(), part.lastAppliedUpdate());
+
+ cpRec.addCacheGroupState(grp.groupId(), state);
+ }
+
+ if (curr.nextSnapshot)
+ snapshotMgr.onMarkCheckPointBegin(curr.snapshotOperation, map);
+
+ IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> tup = beginAllCheckpoints();
+
+ // Todo it maybe more optimally
+ Collection<FullPageId> cpPagesList = new ArrayList<>(tup.get2());
+
+ for (GridMultiCollectionWrapper<FullPageId> col : tup.get1()) {
+ for (int i = 0; i < col.collectionsSize(); i++)
+ cpPagesList.addAll(col.innerCollection(i));
+ }
+
+ cpPages = new GridMultiCollectionWrapper<>(cpPagesList);
+
+ if (!F.isEmpty(cpPages)) {
+ // No page updates for this checkpoint are allowed from now on.
+ cpPtr = cctx.wal().log(cpRec);
+
+ if (cpPtr == null)
+ cpPtr = CheckpointStatus.NULL_PTR;
+ }
+ }
+ finally {
+ checkpointLock.writeLock().unlock();
+
+ tracker.onLockRelease();
+ }
+
+ curr.cpBeginFut.onDone();
+
+ if (!F.isEmpty(cpPages)) {
+ assert cpPtr != null;
+
+ // Sync log outside the checkpoint write lock.
+ cctx.wal().fsync(cpPtr);
+
+ long cpTs = System.currentTimeMillis();
+
+ CheckpointEntry cpEntry = writeCheckpointEntry(
+ tmpWriteBuf,
+ cpTs,
+ cpRec.checkpointId(),
+ cpPtr,
+ cpRec,
+ CheckpointEntryType.START);
+
+ checkpointHist.addCheckpointEntry(cpEntry);
+
+ if (printCheckpointStats)
+ if (log.isInfoEnabled())
+ log.info(String.format("Checkpoint started [checkpointId=%s, startPtr=%s, checkpointLockWait=%dms, " +
+ "checkpointLockHoldTime=%dms, pages=%d, reason='%s']",
+ cpRec.checkpointId(),
+ cpPtr,
+ tracker.lockWaitDuration(),
+ tracker.lockHoldDuration(),
+ cpPages.size(),
+ curr.reason)
+ );
+
+ return new Checkpoint(cpEntry, cpPages, curr);
+ }
+ else {
+ if (printCheckpointStats) {
+ if (log.isInfoEnabled())
+ LT.info(log, String.format("Skipping checkpoint (no pages were modified) [" +
+ "checkpointLockWait=%dms, checkpointLockHoldTime=%dms, reason='%s']",
+ tracker.lockWaitDuration(),
+ tracker.lockHoldDuration(),
+ curr.reason));
+ }
+
+ return new Checkpoint(null, null, curr);
+ }
+ }
+
+ /**
+ * @return tuple with collections of FullPageIds obtained from each PageMemory and overall number of dirty pages.
+ */
+ private IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> beginAllCheckpoints() {
+ Collection<GridMultiCollectionWrapper<FullPageId>> res = new ArrayList(memoryPolicies().size());
+
+ int pagesNum = 0;
+
+ for (MemoryPolicy memPlc : memoryPolicies()) {
+ GridMultiCollectionWrapper<FullPageId> nextCpPagesCol = ((PageMemoryEx) memPlc.pageMemory()).beginCheckpoint();
+
+ pagesNum += nextCpPagesCol.size();
+
+ res.add(nextCpPagesCol);
+ }
+
+ return new IgniteBiTuple<>(res, pagesNum);
+ }
+
+ /**
+ * @param chp Checkpoint snapshot.
+ */
+ private void markCheckpointEnd(Checkpoint chp) throws IgniteCheckedException {
+ synchronized (this) {
+ for (MemoryPolicy memPlc : memoryPolicies())
+ ((PageMemoryEx)memPlc.pageMemory()).finishCheckpoint();
+
+ if (chp.hasDelta())
+ writeCheckpointEntry(
+ tmpWriteBuf,
+ chp.cpEntry.checkpointTimestamp(),
+ chp.cpEntry.checkpointId(),
+ chp.cpEntry.checkpointMark(),
+ null,
+ CheckpointEntryType.END);
+ }
+
+ checkpointHist.onCheckpointFinished(chp);
+
+ chp.progress.cpFinishFut.onDone();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void cancel() {
+ if (log.isDebugEnabled())
+ log.debug("Cancelling grid runnable: " + this);
+
+ // Do not interrupt runner thread.
+ isCancelled = true;
+
+ synchronized (this) {
+ notifyAll();
+ }
+ }
+
+ /**
+ *
+ */
+ public void shutdownNow() {
+ shutdownNow = true;
+
+ if (!isCancelled)
+ cancel();
+ }
+ }
+
+ /**
+ *
+ */
+ private class WriteCheckpointPages implements Runnable {
+ /** */
+ private CheckpointMetricsTracker tracker;
+
+ /** */
+ private Collection<FullPageId> writePageIds;
+
+ /** */
+ private GridConcurrentHashSet<PageStore> updStores;
+
+ /** */
+ private CountDownFuture doneFut;
+
+ /**
+ * @param writePageIds Write page IDs.
+ */
+ private WriteCheckpointPages(
+ CheckpointMetricsTracker tracker,
+ Collection<FullPageId> writePageIds,
+ GridConcurrentHashSet<PageStore> updStores,
+ CountDownFuture doneFut
+ ) {
+ this.tracker = tracker;
+ this.writePageIds = writePageIds;
+ this.updStores = updStores;
+ this.doneFut = doneFut;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ ByteBuffer tmpWriteBuf = threadBuf.get();
+
+ long writeAddr = GridUnsafe.bufferAddress(tmpWriteBuf);
+
+ snapshotMgr.beforeCheckpointPageWritten();
+
+ try {
+ for (FullPageId fullId : writePageIds) {
+ if (checkpointer.shutdownNow)
+ break;
+
+ tmpWriteBuf.rewind();
+
+ snapshotMgr.beforePageWrite(fullId);
+
+ int grpId = fullId.cacheId();
+
+ CacheGroupContext grp = context().cache().cacheGroup(grpId);
+
+ if (grp == null)
+ continue;
+
+ PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
+
+ Integer tag = pageMem.getForCheckpoint(fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
+
+ if (tag != null) {
+ tmpWriteBuf.rewind();
+
+ if (persStoreMetrics.metricsEnabled()) {
+ int pageType = PageIO.getType(tmpWriteBuf);
+
+ if (PageIO.isDataPageType(pageType))
+ tracker.onDataPageWritten();
+ }
+
+ if (!skipCrc) {
+ PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize()));
+
+ tmpWriteBuf.rewind();
+ }
+
+ snapshotMgr.onPageWrite(fullId, tmpWriteBuf);
+
+ tmpWriteBuf.rewind();
+
+ PageIO.setCrc(writeAddr, 0);
+
+
<TRUNCATED>
[32/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
new file mode 100644
index 0000000..74d251a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java
@@ -0,0 +1,4808 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree;
+
+import java.io.Externalizable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.FixCountRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.FixLeftmostChildRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.FixRemoveId;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InsertRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageAddRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageCutRootRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRootInlineRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.NewRootInitRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.RemoveRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.ReplaceRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.SplitExistingPageRecord;
+import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.GridArrays;
+import org.apache.ignite.internal.util.GridLongList;
+import org.apache.ignite.internal.util.IgniteTree;
+import org.apache.ignite.internal.util.lang.GridCursor;
+import org.apache.ignite.internal.util.lang.GridTreePrinter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.SB;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.DONE;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.FALSE;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.READY;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Bool.TRUE;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Result.FOUND;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Result.GO_DOWN;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Result.GO_DOWN_X;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Result.NOT_FOUND;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Result.RETRY;
+import static org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree.Result.RETRY_ROOT;
+
+/**
+ * Abstract B+Tree.
+ */
+@SuppressWarnings({"RedundantThrowsDeclaration", "ConstantValueVariableUse"})
+public abstract class BPlusTree<L, T extends L> extends DataStructure implements IgniteTree<L, T> {
+ /** */
+ private static final Object[] EMPTY = {};
+
+ /** */
+ private static volatile boolean interrupted;
+
+ /** */
+ private final AtomicBoolean destroyed = new AtomicBoolean(false);
+
+ /** */
+ private final String name;
+
+ /** */
+ private final float minFill;
+
+ /** */
+ private final float maxFill;
+
+ /** */
+ protected final long metaPageId;
+
+ /** */
+ private boolean canGetRowFromInner;
+
+ /** */
+ private IOVersions<? extends BPlusInnerIO<L>> innerIos;
+
+ /** */
+ private IOVersions<? extends BPlusLeafIO<L>> leafIos;
+
+ /** */
+ private final AtomicLong globalRmvId;
+
+ /** */
+ private volatile TreeMetaData treeMeta;
+
+ /** */
+ private final GridTreePrinter<Long> treePrinter = new GridTreePrinter<Long>() {
+ /** */
+ private boolean keys = true;
+
+ @Override protected List<Long> getChildren(final Long pageId) {
+ if (pageId == null || pageId == 0L)
+ return null;
+
+ try {
+ long page = acquirePage(pageId);
+
+ try {
+ long pageAddr = readLock(pageId, page); // No correctness guaranties.
+
+ try {
+ BPlusIO io = io(pageAddr);
+
+ if (io.isLeaf())
+ return null;
+
+ int cnt = io.getCount(pageAddr);
+
+ assert cnt >= 0 : cnt;
+
+ List<Long> res;
+
+ if (cnt > 0) {
+ res = new ArrayList<>(cnt + 1);
+
+ for (int i = 0; i < cnt; i++)
+ res.add(inner(io).getLeft(pageAddr, i));
+
+ res.add(inner(io).getRight(pageAddr, cnt - 1));
+ }
+ else {
+ long left = inner(io).getLeft(pageAddr, 0);
+
+ res = left == 0 ? Collections.<Long>emptyList() : Collections.singletonList(left);
+ }
+
+ return res;
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+ catch (IgniteCheckedException ignored) {
+ throw new AssertionError("Can not acquire page.");
+ }
+ }
+
+ @Override protected String formatTreeNode(final Long pageId) {
+ if (pageId == null)
+ return ">NPE<";
+
+ if (pageId == 0L)
+ return "<Zero>";
+
+ try {
+ long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page); // No correctness guaranties.
+ try {
+ BPlusIO<L> io = io(pageAddr);
+
+ return printPage(io, pageAddr, keys);
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ };
+
+ /** */
+ private final GetPageHandler<Get> askNeighbor = new AskNeighbor();
+
+ /**
+ *
+ */
+ private class AskNeighbor extends GetPageHandler<Get> {
+ /** {@inheritDoc} */
+ @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Get g, int isBack) {
+ assert !io.isLeaf(); // Inner page.
+
+ boolean back = isBack == TRUE.ordinal();
+
+ long res = doAskNeighbor(io, pageAddr, back);
+
+ if (back) {
+ if (io.getForward(pageAddr) != g.backId) // See how g.backId is setup in removeDown for this check.
+ return RETRY;
+
+ g.backId(res);
+ }
+ else {
+ assert isBack == FALSE.ordinal() : isBack;
+
+ g.fwdId(res);
+ }
+
+ return FOUND;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Get> search = new Search();
+
+ /**
+ *
+ */
+ private class Search extends GetPageHandler<Get> {
+ /** {@inheritDoc} */
+ @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Get g, int lvl)
+ throws IgniteCheckedException {
+ // Check the triangle invariant.
+ if (io.getForward(pageAddr) != g.fwdId)
+ return RETRY;
+
+ boolean needBackIfRouting = g.backId != 0;
+
+ g.backId(0L); // Usually we'll go left down and don't need it.
+
+ int cnt = io.getCount(pageAddr);
+
+ int idx;
+
+ if (g.findLast)
+ idx = io.isLeaf() ? cnt - 1 : -cnt - 1; // (-cnt - 1) mimics not_found result of findInsertionPoint
+ // in case of cnt = 0 we end up in 'not found' branch below with idx being 0 after fix() adjustment
+ else
+ idx = findInsertionPoint(lvl, io, pageAddr, 0, cnt, g.row, g.shift);
+
+ boolean found = idx >= 0;
+
+ if (found) { // Found exact match.
+ assert g.getClass() != GetCursor.class;
+
+ if (g.found(io, pageAddr, idx, lvl))
+ return FOUND;
+
+ // Else we need to reach leaf page, go left down.
+ }
+ else {
+ idx = fix(idx);
+
+ if (g.notFound(io, pageAddr, idx, lvl)) // No way down, stop here.
+ return NOT_FOUND;
+ }
+
+ assert !io.isLeaf() : io;
+
+ // If idx == cnt then we go right down, else left down: getLeft(cnt) == getRight(cnt - 1).
+ g.pageId(inner(io).getLeft(pageAddr, idx));
+
+ // If we see the tree in consistent state, then our right down page must be forward for our left down page,
+ // we need to setup fwdId and/or backId to be able to check this invariant on lower level.
+ if (idx < cnt) {
+ // Go left down here.
+ g.fwdId(inner(io).getRight(pageAddr, idx));
+ }
+ else {
+ // Go right down here or it is an empty branch.
+ assert idx == cnt;
+
+ // Here child's forward is unknown to us (we either go right or it is an empty "routing" page),
+ // need to ask our forward about the child's forward (it must be leftmost child of our forward page).
+ // This is ok from the locking standpoint because we take all locks in the forward direction.
+ long fwdId = io.getForward(pageAddr);
+
+ // Setup fwdId.
+ if (fwdId == 0)
+ g.fwdId(0L);
+ else {
+ // We can do askNeighbor on forward page here because we always take locks in forward direction.
+ Result res = askNeighbor(fwdId, g, false);
+
+ if (res != FOUND)
+ return res; // Retry.
+ }
+
+ // Setup backId.
+ if (cnt != 0) // It is not a routing page and we are going to the right, can get backId here.
+ g.backId(inner(io).getLeft(pageAddr, cnt - 1));
+ else if (needBackIfRouting) {
+ // Can't get backId here because of possible deadlock and it is only needed for remove operation.
+ return GO_DOWN_X;
+ }
+ }
+
+ return GO_DOWN;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Put> replace = new Replace();
+
+ /**
+ *
+ */
+ private class Replace extends GetPageHandler<Put> {
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Put p, int lvl)
+ throws IgniteCheckedException {
+ // Check the triangle invariant.
+ if (io.getForward(pageAddr) != p.fwdId)
+ return RETRY;
+
+ assert p.btmLvl == 0 : "split is impossible with replace";
+
+ final int cnt = io.getCount(pageAddr);
+ final int idx = findInsertionPoint(lvl, io, pageAddr, 0, cnt, p.row, 0);
+
+ if (idx < 0) // Not found, split or merge happened.
+ return RETRY;
+
+ // Replace link at idx with new one.
+ // Need to read link here because `p.finish()` will clear row.
+ L newRow = p.row;
+
+ // Detach the old row if we are on a leaf page.
+ if (lvl == 0) {
+ assert p.oldRow == null; // The old row must be set only once.
+
+ // Inner replace state must be consistent by the end of the operation.
+ assert p.needReplaceInner == FALSE || p.needReplaceInner == DONE : p.needReplaceInner;
+
+ // Need to replace inner key if now we are replacing the rightmost row and have a forward page.
+ if (canGetRowFromInner && idx + 1 == cnt && p.fwdId != 0L && p.needReplaceInner == FALSE) {
+ // Can happen only for invoke, otherwise inner key must be replaced on the way down.
+ assert p.invoke != null;
+
+ // We need to restart the operation from root to perform inner replace.
+ // On the second pass we will not get here (will avoid infinite loop) because
+ // needReplaceInner will be DONE or our key will not be the rightmost anymore.
+ return RETRY_ROOT;
+ }
+
+ // Get old row in leaf page to reduce contention at upper level.
+ p.oldRow = p.needOld ? getRow(io, pageAddr, idx) : (T)Boolean.TRUE;
+
+ p.finish();
+ }
+
+ boolean needWal = needWalDeltaRecord(pageId, page, null);
+
+ byte[] newRowBytes = io.store(pageAddr, idx, newRow, null, needWal);
+
+ if (needWal)
+ wal.log(new ReplaceRecord<>(cacheId, pageId, io, newRowBytes, idx));
+
+ return FOUND;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Put> insert = new Insert();
+
+ /**
+ *
+ */
+ private class Insert extends GetPageHandler<Put> {
+ /** {@inheritDoc} */
+ @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Put p, int lvl)
+ throws IgniteCheckedException {
+ assert p.btmLvl == lvl : "we must always insert at the bottom level: " + p.btmLvl + " " + lvl;
+
+ // Check triangle invariant.
+ if (io.getForward(pageAddr) != p.fwdId)
+ return RETRY;
+
+ int cnt = io.getCount(pageAddr);
+ int idx = findInsertionPoint(lvl, io, pageAddr, 0, cnt, p.row, 0);
+
+ if (idx >= 0) // We do not support concurrent put of the same key.
+ throw new IllegalStateException("Duplicate row in index.");
+
+ idx = fix(idx);
+
+ // Do insert.
+ L moveUpRow = p.insert(pageId, page, pageAddr, io, idx, lvl);
+
+ // Check if split happened.
+ if (moveUpRow != null) {
+ p.btmLvl++; // Get high.
+ p.row = moveUpRow;
+
+ if (p.invoke != null)
+ p.invoke.row = moveUpRow;
+
+ // Here forward page can't be concurrently removed because we keep write lock on tail which is the only
+ // page who knows about the forward page, because it was just produced by split.
+ p.rightId = io.getForward(pageAddr);
+ p.tail(pageId, page, pageAddr);
+
+ assert p.rightId != 0;
+ }
+ else
+ p.finish();
+
+ return FOUND;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Remove> rmvFromLeaf = new RemoveFromLeaf();
+
+ /**
+ *
+ */
+ private class RemoveFromLeaf extends GetPageHandler<Remove> {
+ /** {@inheritDoc} */
+ @Override public Result run0(long leafId, long leafPage, long leafAddr, BPlusIO<L> io, Remove r, int lvl)
+ throws IgniteCheckedException {
+ assert lvl == 0 : lvl; // Leaf.
+
+ // Check the triangle invariant.
+ if (io.getForward(leafAddr) != r.fwdId)
+ return RETRY;
+
+ final int cnt = io.getCount(leafAddr);
+
+ assert cnt <= Short.MAX_VALUE: cnt;
+
+ int idx = findInsertionPoint(lvl, io, leafAddr, 0, cnt, r.row, 0);
+
+ if (idx < 0)
+ return RETRY; // We've found exact match on search but now it's gone.
+
+ assert idx >= 0 && idx < cnt: idx;
+
+ // Need to do inner replace when we remove the rightmost element and the leaf have no forward page,
+ // i.e. it is not the rightmost leaf of the tree.
+ boolean needReplaceInner = canGetRowFromInner && idx == cnt - 1 && io.getForward(leafAddr) != 0;
+
+ // !!! Before modifying state we have to make sure that we will not go for retry.
+
+ // We may need to replace inner key or want to merge this leaf with sibling after the remove -> keep lock.
+ if (needReplaceInner ||
+ // We need to make sure that we have back or forward to be able to merge.
+ ((r.fwdId != 0 || r.backId != 0) && mayMerge(cnt - 1, io.getMaxCount(leafAddr, pageSize())))) {
+ // If we have backId then we've already locked back page, nothing to do here.
+ if (r.fwdId != 0 && r.backId == 0) {
+ Result res = r.lockForward(0);
+
+ if (res != FOUND) {
+ assert r.tail == null;
+
+ return res; // Retry.
+ }
+
+ assert r.tail != null; // We've just locked forward page.
+ }
+
+ // Retry must reset these fields when we release the whole branch without remove.
+ assert r.needReplaceInner == FALSE: "needReplaceInner";
+ assert r.needMergeEmptyBranch == FALSE: "needMergeEmptyBranch";
+
+ if (cnt == 1) // It was the last element on the leaf.
+ r.needMergeEmptyBranch = TRUE;
+
+ if (needReplaceInner)
+ r.needReplaceInner = TRUE;
+
+ Tail<L> t = r.addTail(leafId, leafPage, leafAddr, io, 0, Tail.EXACT);
+
+ t.idx = (short)idx;
+
+ // We will do the actual remove only when we made sure that
+ // we've locked the whole needed branch correctly.
+ return FOUND;
+ }
+
+ r.removeDataRowFromLeaf(leafId, leafPage, leafAddr, null, io, cnt, idx);
+
+ return FOUND;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Remove> lockBackAndRmvFromLeaf = new LockBackAndRmvFromLeaf();
+
+ /**
+ *
+ */
+ private class LockBackAndRmvFromLeaf extends GetPageHandler<Remove> {
+ /** {@inheritDoc} */
+ @Override protected Result run0(long backId, long backPage, long backAddr, BPlusIO<L> io, Remove r, int lvl)
+ throws IgniteCheckedException {
+ // Check that we have consistent view of the world.
+ if (io.getForward(backAddr) != r.pageId)
+ return RETRY;
+
+ // Correct locking order: from back to forward.
+ Result res = r.doRemoveFromLeaf();
+
+ // Keep locks on back and leaf pages for subsequent merges.
+ if (res == FOUND && r.tail != null)
+ r.addTail(backId, backPage, backAddr, io, lvl, Tail.BACK);
+
+ return res;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Remove> lockBackAndTail = new LockBackAndTail();
+
+ /**
+ *
+ */
+ private class LockBackAndTail extends GetPageHandler<Remove> {
+ /** {@inheritDoc} */
+ @Override public Result run0(long backId, long backPage, long backAddr, BPlusIO<L> io, Remove r, int lvl)
+ throws IgniteCheckedException {
+ // Check that we have consistent view of the world.
+ if (io.getForward(backAddr) != r.pageId)
+ return RETRY;
+
+ // Correct locking order: from back to forward.
+ Result res = r.doLockTail(lvl);
+
+ if (res == FOUND)
+ r.addTail(backId, backPage, backAddr, io, lvl, Tail.BACK);
+
+ return res;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Remove> lockTailForward = new LockTailForward();
+
+ /**
+ *
+ */
+ private class LockTailForward extends GetPageHandler<Remove> {
+ /** {@inheritDoc} */
+ @Override protected Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Remove r, int lvl)
+ throws IgniteCheckedException {
+ r.addTail(pageId, page, pageAddr, io, lvl, Tail.FORWARD);
+
+ return FOUND;
+ }
+ }
+
+ /** */
+ private final GetPageHandler<Remove> lockTail = new LockTail();
+
+ /**
+ *
+ */
+ private class LockTail extends GetPageHandler<Remove> {
+ /** {@inheritDoc} */
+ @Override public Result run0(long pageId, long page, long pageAddr, BPlusIO<L> io, Remove r, int lvl)
+ throws IgniteCheckedException {
+ assert lvl > 0 : lvl; // We are not at the bottom.
+
+ // Check that we have a correct view of the world.
+ if (io.getForward(pageAddr) != r.fwdId)
+ return RETRY;
+
+ // We don't have a back page, need to lock our forward and become a back for it.
+ if (r.fwdId != 0 && r.backId == 0) {
+ Result res = r.lockForward(lvl);
+
+ if (res != FOUND)
+ return res; // Retry.
+ }
+
+ r.addTail(pageId, page, pageAddr, io, lvl, Tail.EXACT);
+
+ return FOUND;
+ }
+ }
+
+ /** */
+ private final PageHandler<Void, Bool> cutRoot = new CutRoot();
+
+ /**
+ *
+ */
+ private class CutRoot extends PageHandler<Void, Bool> {
+ /** {@inheritDoc} */
+ @Override public Bool run(int cacheId, long metaId, long metaPage, long metaAddr, PageIO iox, Boolean walPlc, Void ignore, int lvl)
+ throws IgniteCheckedException {
+ // Safe cast because we should never recycle meta page until the tree is destroyed.
+ BPlusMetaIO io = (BPlusMetaIO)iox;
+
+ assert lvl == io.getRootLevel(metaAddr); // Can drop only root.
+
+ io.cutRoot(metaAddr, pageSize());
+
+ if (needWalDeltaRecord(metaId, metaPage, walPlc))
+ wal.log(new MetaPageCutRootRecord(cacheId, metaId));
+
+ int newLvl = lvl - 1;
+
+ assert io.getRootLevel(metaAddr) == newLvl;
+
+ treeMeta = new TreeMetaData(newLvl, io.getFirstPageId(metaAddr, newLvl));
+
+ return TRUE;
+ }
+ }
+
+ /** */
+ private final PageHandler<Long, Bool> addRoot = new AddRoot();
+
+ /**
+ *
+ */
+ private class AddRoot extends PageHandler<Long, Bool> {
+ /** {@inheritDoc} */
+ @Override public Bool run(int cacheId, long metaId, long metaPage, long pageAddr, PageIO iox, Boolean walPlc, Long rootPageId, int lvl)
+ throws IgniteCheckedException {
+ assert rootPageId != null;
+
+ // Safe cast because we should never recycle meta page until the tree is destroyed.
+ BPlusMetaIO io = (BPlusMetaIO)iox;
+
+ assert lvl == io.getLevelsCount(pageAddr);
+
+ io.addRoot(pageAddr, rootPageId, pageSize());
+
+ if (needWalDeltaRecord(metaId, metaPage, walPlc))
+ wal.log(new MetaPageAddRootRecord(cacheId, metaId, rootPageId));
+
+ assert io.getRootLevel(pageAddr) == lvl;
+ assert io.getFirstPageId(pageAddr, lvl) == rootPageId;
+
+ treeMeta = new TreeMetaData(lvl, rootPageId);
+
+ return TRUE;
+ }
+ }
+
+ /** */
+ private final PageHandler<Long, Bool> initRoot = new InitRoot();
+
+ /**
+ *
+ */
+ private class InitRoot extends PageHandler<Long, Bool> {
+ /** {@inheritDoc} */
+ @Override public Bool run(int cacheId, long metaId, long metaPage, long pageAddr, PageIO iox, Boolean walPlc, Long rootId, int inlineSize)
+ throws IgniteCheckedException {
+ assert rootId != null;
+
+ // Safe cast because we should never recycle meta page until the tree is destroyed.
+ BPlusMetaIO io = (BPlusMetaIO)iox;
+
+ io.initRoot(pageAddr, rootId, pageSize());
+ io.setInlineSize(pageAddr, inlineSize);
+
+ if (needWalDeltaRecord(metaId, metaPage, walPlc))
+ wal.log(new MetaPageInitRootInlineRecord(cacheId, metaId, rootId, inlineSize));
+
+ assert io.getRootLevel(pageAddr) == 0;
+ assert io.getFirstPageId(pageAddr, 0) == rootId;
+
+ treeMeta = new TreeMetaData(0, rootId);
+
+ return TRUE;
+ }
+ }
+
+ /**
+ * @param name Tree name.
+ * @param cacheId Cache ID.
+ * @param pageMem Page memory.
+ * @param wal Write ahead log manager.
+ * @param globalRmvId Remove ID.
+ * @param metaPageId Meta page ID.
+ * @param reuseList Reuse list.
+ * @param innerIos Inner IO versions.
+ * @param leafIos Leaf IO versions.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected BPlusTree(
+ String name,
+ int cacheId,
+ PageMemory pageMem,
+ IgniteWriteAheadLogManager wal,
+ AtomicLong globalRmvId,
+ long metaPageId,
+ ReuseList reuseList,
+ IOVersions<? extends BPlusInnerIO<L>> innerIos,
+ IOVersions<? extends BPlusLeafIO<L>> leafIos
+ ) throws IgniteCheckedException {
+ this(name, cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList);
+ setIos(innerIos, leafIos);
+ }
+
+ /**
+ * @param name Tree name.
+ * @param cacheId Cache ID.
+ * @param pageMem Page memory.
+ * @param wal Write ahead log manager.
+ * @param globalRmvId Remove ID.
+ * @param metaPageId Meta page ID.
+ * @param reuseList Reuse list.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected BPlusTree(
+ String name,
+ int cacheId,
+ PageMemory pageMem,
+ IgniteWriteAheadLogManager wal,
+ AtomicLong globalRmvId,
+ long metaPageId,
+ ReuseList reuseList
+ ) throws IgniteCheckedException {
+ super(cacheId, pageMem, wal);
+
+ assert !F.isEmpty(name);
+
+ // TODO make configurable: 0 <= minFill <= maxFill <= 1
+ minFill = 0f; // Testing worst case when merge happens only on empty page.
+ maxFill = 0f; // Avoiding random effects on testing.
+
+ assert metaPageId != 0L;
+
+ this.metaPageId = metaPageId;
+ this.name = name;
+ this.reuseList = reuseList;
+ this.globalRmvId = globalRmvId;
+ }
+
+ /**
+ * @param innerIos Inner IO versions.
+ * @param leafIos Leaf IO versions.
+ */
+ protected void setIos(IOVersions<? extends BPlusInnerIO<L>> innerIos,
+ IOVersions<? extends BPlusLeafIO<L>> leafIos) {
+ assert innerIos != null;
+ assert leafIos != null;
+
+ this.canGetRowFromInner = innerIos.latest().canGetRow(); // TODO refactor
+ this.innerIos = innerIos;
+ this.leafIos = leafIos;
+ }
+
+ /**
+ * @return Tree name.
+ */
+ public final String getName() {
+ return name;
+ }
+
+ /**
+ * Initialize new tree.
+ *
+ * @param initNew {@code True} if new tree should be created.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final void initTree(boolean initNew) throws IgniteCheckedException {
+ initTree(initNew, 0);
+ }
+
+ /**
+ * Initialize new tree.
+ *
+ * @param initNew {@code True} if new tree should be created.
+ * @param inlineSize Inline size.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected final void initTree(boolean initNew, int inlineSize) throws IgniteCheckedException {
+ if (initNew) {
+ // Allocate the first leaf page, it will be our root.
+ long rootId = allocatePage(null);
+
+ init(rootId, latestLeafIO());
+
+ // Initialize meta page with new root page.
+ Bool res = write(metaPageId, initRoot, BPlusMetaIO.VERSIONS.latest(), rootId, inlineSize, FALSE);
+
+ assert res == TRUE: res;
+
+ assert treeMeta != null;
+ }
+ }
+
+ /**
+ * @return Tree meta data.
+ * @throws IgniteCheckedException If failed.
+ */
+ private TreeMetaData treeMeta() throws IgniteCheckedException {
+ TreeMetaData meta0 = treeMeta;
+
+ if (meta0 != null)
+ return meta0;
+
+ final long metaPage = acquirePage(metaPageId);
+ try {
+ long pageAddr = readLock(metaPageId, metaPage); // Meta can't be removed.
+
+ assert pageAddr != 0 : "Failed to read lock meta page [metaPageId=" +
+ U.hexLong(metaPageId) + ']';
+
+ try {
+ BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(pageAddr);
+
+ int rootLvl = io.getRootLevel(pageAddr);
+ long rootId = io.getFirstPageId(pageAddr, rootLvl);
+
+ treeMeta = meta0 = new TreeMetaData(rootLvl, rootId);
+ }
+ finally {
+ readUnlock(metaPageId, metaPage, pageAddr);
+ }
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+
+ return meta0;
+ }
+
+ /**
+ * @return Root level.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int getRootLevel() throws IgniteCheckedException {
+ TreeMetaData meta0 = treeMeta();
+
+ assert meta0 != null;
+
+ return meta0.rootLvl;
+ }
+
+ /**
+ * @param metaId Meta page ID.
+ * @param metaPage Meta page pointer.
+ * @param lvl Level, if {@code 0} then it is a bottom level, if negative then root.
+ * @return Page ID.
+ */
+ private long getFirstPageId(long metaId, long metaPage, int lvl) {
+ long pageAddr = readLock(metaId, metaPage); // Meta can't be removed.
+
+ try {
+ BPlusMetaIO io = BPlusMetaIO.VERSIONS.forPage(pageAddr);
+
+ if (lvl < 0)
+ lvl = io.getRootLevel(pageAddr);
+
+ if (lvl >= io.getLevelsCount(pageAddr))
+ return 0;
+
+ return io.getFirstPageId(pageAddr, lvl);
+ }
+ finally {
+ readUnlock(metaId, metaPage, pageAddr);
+ }
+ }
+
+ /**
+ * @param upper Upper bound.
+ * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row.
+ * @return Cursor.
+ * @throws IgniteCheckedException If failed.
+ */
+ private GridCursor<T> findLowerUnbounded(L upper, Object x) throws IgniteCheckedException {
+ ForwardCursor cursor = new ForwardCursor(null, upper, x);
+
+ long firstPageId;
+
+ long metaPage = acquirePage(metaPageId);
+ try {
+ firstPageId = getFirstPageId(metaPageId, metaPage, 0); // Level 0 is always at the bottom.
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+
+ long firstPage = acquirePage(firstPageId);
+
+ try {
+ long pageAddr = readLock(firstPageId, firstPage); // We always merge pages backwards, the first page is never removed.
+
+ try {
+ cursor.init(pageAddr, io(pageAddr), 0);
+ }
+ finally {
+ readUnlock(firstPageId, firstPage, pageAddr);
+ }
+ }
+ finally {
+ releasePage(firstPageId, firstPage);
+ }
+
+ return cursor;
+ }
+
+ /**
+ * Check if the tree is getting destroyed.
+ */
+ private void checkDestroyed() {
+ if (destroyed.get())
+ throw new IllegalStateException("Tree is being concurrently destroyed: " + getName());
+ }
+
+ /**
+ * @param lower Lower bound inclusive or {@code null} if unbounded.
+ * @param upper Upper bound inclusive or {@code null} if unbounded.
+ * @return Cursor.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public GridCursor<T> find(L lower, L upper) throws IgniteCheckedException {
+ return find(lower, upper, null);
+ }
+
+ /**
+ * @param lower Lower bound inclusive or {@code null} if unbounded.
+ * @param upper Upper bound inclusive or {@code null} if unbounded.
+ * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row.
+ * @return Cursor.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final GridCursor<T> find(L lower, L upper, Object x) throws IgniteCheckedException {
+ checkDestroyed();
+
+ try {
+ if (lower == null)
+ return findLowerUnbounded(upper, x);
+
+ ForwardCursor cursor = new ForwardCursor(lower, upper, x);
+
+ cursor.find();
+
+ return cursor;
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on bounds: [lower=" + lower + ", upper=" + upper + "]", e);
+ }
+ finally {
+ checkDestroyed();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public T findFirst() throws IgniteCheckedException {
+ checkDestroyed();
+
+ try {
+ long firstPageId;
+
+ long metaPage = acquirePage(metaPageId);
+ try {
+ firstPageId = getFirstPageId(metaPageId, metaPage, 0);
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+
+ long page = acquirePage(firstPageId);
+
+ try {
+ long pageAddr = readLock(firstPageId, page);
+
+ try {
+ BPlusIO<L> io = io(pageAddr);
+
+ int cnt = io.getCount(pageAddr);
+
+ if (cnt == 0)
+ return null;
+
+ return getRow(io, pageAddr, 0);
+ }
+ finally {
+ readUnlock(firstPageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(firstPageId, page);
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on first row lookup", e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on first row lookup", e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on first row lookup", e);
+ }
+ finally {
+ checkDestroyed();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override public T findLast() throws IgniteCheckedException {
+ checkDestroyed();
+
+ try {
+ GetOne g = new GetOne(null, null, true);
+ doFind(g);
+
+ return (T)g.row;
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on last row lookup", e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on last row lookup", e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on last row lookup", e);
+ }
+ finally {
+ checkDestroyed();
+ }
+ }
+
+ /**
+ * @param row Lookup row for exact match.
+ * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row.
+ * @return Found result or {@code null}.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public final <R> R findOne(L row, Object x) throws IgniteCheckedException {
+ checkDestroyed();
+
+ try {
+ GetOne g = new GetOne(row, x, false);
+
+ doFind(g);
+
+ return (R)g.row;
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on lookup row: " + row, e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on lookup row: " + row, e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on lookup row: " + row, e);
+ }
+ finally {
+ checkDestroyed();
+ }
+ }
+
+ /**
+ * @param row Lookup row for exact match.
+ * @return Found row.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ @Override public final T findOne(L row) throws IgniteCheckedException {
+ return findOne(row, null);
+ }
+
+ /**
+ * @param g Get.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void doFind(Get g) throws IgniteCheckedException {
+ for (;;) { // Go down with retries.
+ g.init();
+
+ switch (findDown(g, g.rootId, 0L, g.rootLvl)) {
+ case RETRY:
+ case RETRY_ROOT:
+ checkInterrupted();
+
+ continue;
+
+ default:
+ return;
+ }
+ }
+ }
+
+ /**
+ * @param g Get.
+ * @param pageId Page ID.
+ * @param fwdId Expected forward page ID.
+ * @param lvl Level.
+ * @return Result code.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Result findDown(final Get g, final long pageId, final long fwdId, final int lvl)
+ throws IgniteCheckedException {
+ long page = acquirePage(pageId);
+
+ try {
+ for (;;) {
+ // Init args.
+ g.pageId = pageId;
+ g.fwdId = fwdId;
+
+ Result res = read(pageId, page, search, g, lvl, RETRY);
+
+ switch (res) {
+ case GO_DOWN:
+ case GO_DOWN_X:
+ assert g.pageId != pageId;
+ assert g.fwdId != fwdId || fwdId == 0;
+
+ // Go down recursively.
+ res = findDown(g, g.pageId, g.fwdId, lvl - 1);
+
+ switch (res) {
+ case RETRY:
+ checkInterrupted();
+
+ continue; // The child page got split, need to reread our page.
+
+ default:
+ return res;
+ }
+
+ case NOT_FOUND:
+ assert lvl == 0 : lvl;
+
+ g.row = null; // Mark not found result.
+
+ return res;
+
+ default:
+ return res;
+ }
+ }
+ }
+ finally {
+ if (g.canRelease(pageId, lvl))
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param instance Instance name.
+ * @param type Tree type.
+ * @return Tree name.
+ */
+ public static String treeName(String instance, String type) {
+ return instance + "##" + type;
+ }
+
+ /**
+ * For debug.
+ *
+ * @return Tree as {@link String}.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unused")
+ public final String printTree() throws IgniteCheckedException {
+ long rootPageId;
+
+ long metaPage = acquirePage(metaPageId);
+ try {
+ rootPageId = getFirstPageId(metaPageId, metaPage, -1);
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+
+ return treePrinter.print(rootPageId);
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ public final void validateTree() throws IgniteCheckedException {
+ long rootPageId;
+ int rootLvl;
+
+ long metaPage = acquirePage(metaPageId);
+ try {
+ rootLvl = getRootLevel();
+
+ if (rootLvl < 0)
+ fail("Root level: " + rootLvl);
+
+ validateFirstPages(metaPageId, metaPage, rootLvl);
+
+ rootPageId = getFirstPageId(metaPageId, metaPage, rootLvl);
+
+ validateDownPages(rootPageId, 0L, rootLvl);
+
+ validateDownKeys(rootPageId, null, rootLvl);
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param minRow Minimum row.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void validateDownKeys(long pageId, L minRow, int lvl) throws IgniteCheckedException {
+ long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page); // No correctness guaranties.
+
+ try {
+ BPlusIO<L> io = io(pageAddr);
+
+ int cnt = io.getCount(pageAddr);
+
+ if (cnt < 0)
+ fail("Negative count: " + cnt);
+
+ if (io.isLeaf()) {
+ for (int i = 0; i < cnt; i++) {
+ if (minRow != null && compare(lvl, io, pageAddr, i, minRow) <= 0)
+ fail("Wrong sort order: " + U.hexLong(pageId) + " , at " + i + " , minRow: " + minRow);
+
+ minRow = io.getLookupRow(this, pageAddr, i);
+ }
+
+ return;
+ }
+
+ // To find our inner key we have to go left and then always go to the right.
+ for (int i = 0; i < cnt; i++) {
+ L row = io.getLookupRow(this, pageAddr, i);
+
+ if (minRow != null && compare(lvl, io, pageAddr, i, minRow) <= 0)
+ fail("Min row violated: " + row + " , minRow: " + minRow);
+
+ long leftId = inner(io).getLeft(pageAddr, i);
+
+ L leafRow = getGreatestRowInSubTree(leftId);
+
+ int cmp = compare(lvl, io, pageAddr, i, leafRow);
+
+ if (cmp < 0 || (cmp != 0 && canGetRowFromInner))
+ fail("Wrong inner row: " + U.hexLong(pageId) + " , at: " + i + " , leaf: " + leafRow +
+ " , inner: " + row);
+
+ validateDownKeys(leftId, minRow, lvl - 1);
+
+ minRow = row;
+ }
+
+ // Need to handle the rightmost child subtree separately or handle empty routing page.
+ long rightId = inner(io).getLeft(pageAddr, cnt); // The same as getRight(cnt - 1)
+
+ validateDownKeys(rightId, minRow, lvl - 1);
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @return Search row.
+ * @throws IgniteCheckedException If failed.
+ */
+ private L getGreatestRowInSubTree(long pageId) throws IgniteCheckedException {
+ long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page); // No correctness guaranties.
+
+ try {
+ BPlusIO<L> io = io(pageAddr);
+
+ int cnt = io.getCount(pageAddr);
+
+ if (io.isLeaf()) {
+ if (cnt <= 0) // This code is called only if the tree is not empty, so we can't see empty leaf.
+ fail("Invalid leaf count: " + cnt + " " + U.hexLong(pageId));
+
+ return io.getLookupRow(this, pageAddr, cnt - 1);
+ }
+
+ long rightId = inner(io).getLeft(pageAddr, cnt);// The same as getRight(cnt - 1), but good for routing pages.
+
+ return getGreatestRowInSubTree(rightId);
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param metaId Meta page ID.
+ * @param metaPage Meta page pointer.
+ * @param rootLvl Root level.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void validateFirstPages(long metaId, long metaPage, int rootLvl) throws IgniteCheckedException {
+ for (int lvl = rootLvl; lvl > 0; lvl--)
+ validateFirstPage(metaId, metaPage, lvl);
+ }
+
+ /**
+ * @param msg Message.
+ */
+ private static void fail(Object msg) {
+ throw new AssertionError(msg);
+ }
+
+ /**
+ * @param metaId Meta page ID.
+ * @param metaPage Meta page pointer.
+ * @param lvl Level.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void validateFirstPage(long metaId, long metaPage, int lvl) throws IgniteCheckedException {
+ if (lvl == 0)
+ fail("Leaf level: " + lvl);
+
+ long pageId = getFirstPageId(metaId, metaPage, lvl);
+
+ long leftmostChildId;
+
+ long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page); // No correctness guaranties.
+
+ try {
+ BPlusIO<L> io = io(pageAddr);
+
+ if (io.isLeaf())
+ fail("Leaf.");
+
+ leftmostChildId = inner(io).getLeft(pageAddr, 0);
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+
+ long firstDownPageId = getFirstPageId(metaId, metaPage, lvl - 1);
+
+ if (firstDownPageId != leftmostChildId)
+ fail(new SB("First: meta ").appendHex(firstDownPageId).a(", child ").appendHex(leftmostChildId));
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param fwdId Forward ID.
+ * @param lvl Level.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void validateDownPages(long pageId, long fwdId, int lvl) throws IgniteCheckedException {
+ long page = acquirePage(pageId);
+ try {
+ long pageAddr = readLock(pageId, page); // No correctness guaranties.
+
+ try {
+ long realPageId = BPlusIO.getPageId(pageAddr);
+
+ if (realPageId != pageId)
+ fail(new SB("ABA on page ID: ref ").appendHex(pageId).a(", buf ").appendHex(realPageId));
+
+ BPlusIO<L> io = io(pageAddr);
+
+ if (io.isLeaf() != (lvl == 0)) // Leaf pages only at the level 0.
+ fail("Leaf level mismatch: " + lvl);
+
+ long actualFwdId = io.getForward(pageAddr);
+
+ if (actualFwdId != fwdId)
+ fail(new SB("Triangle: expected fwd ").appendHex(fwdId).a(", actual fwd ").appendHex(actualFwdId));
+
+ int cnt = io.getCount(pageAddr);
+
+ if (cnt < 0)
+ fail("Negative count: " + cnt);
+
+ if (io.isLeaf()) {
+ if (cnt == 0 && getRootLevel() != 0)
+ fail("Empty leaf page.");
+ }
+ else {
+ // Recursively go down if we are on inner level.
+ for (int i = 0; i < cnt; i++)
+ validateDownPages(inner(io).getLeft(pageAddr, i), inner(io).getRight(pageAddr, i), lvl - 1);
+
+ if (fwdId != 0) {
+ // For the rightmost child ask neighbor.
+ long fwdId0 = fwdId;
+ long fwdPage = acquirePage(fwdId0);
+ try {
+ long fwdPageAddr = readLock(fwdId0, fwdPage); // No correctness guaranties.
+
+ try {
+ if (io(fwdPageAddr) != io)
+ fail("IO on the same level must be the same");
+
+ fwdId = inner(io).getLeft(fwdPageAddr, 0);
+ }
+ finally {
+ readUnlock(fwdId0, fwdPage, fwdPageAddr);
+ }
+ }
+ finally {
+ releasePage(fwdId0, fwdPage);
+ }
+ }
+
+ long leftId = inner(io).getLeft(pageAddr, cnt); // The same as io.getRight(cnt - 1) but works for routing pages.
+
+ validateDownPages(leftId, fwdId, lvl - 1);
+ }
+ }
+ finally {
+ readUnlock(pageId, page, pageAddr);
+ }
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param io IO.
+ * @param pageAddr Page address.
+ * @param keys Keys.
+ * @return String.
+ * @throws IgniteCheckedException If failed.
+ */
+ private String printPage(BPlusIO<L> io, long pageAddr, boolean keys) throws IgniteCheckedException {
+ StringBuilder b = new StringBuilder();
+
+ b.append(formatPageId(PageIO.getPageId(pageAddr)));
+
+ b.append(" [ ");
+ b.append(io.isLeaf() ? "L " : "I ");
+
+ int cnt = io.getCount(pageAddr);
+ long fwdId = io.getForward(pageAddr);
+
+ b.append("cnt=").append(cnt).append(' ');
+ b.append("fwd=").append(formatPageId(fwdId)).append(' ');
+
+ if (!io.isLeaf()) {
+ b.append("lm=").append(formatPageId(inner(io).getLeft(pageAddr, 0))).append(' ');
+
+ if (cnt > 0)
+ b.append("rm=").append(formatPageId(inner(io).getRight(pageAddr, cnt - 1))).append(' ');
+ }
+
+ if (keys)
+ b.append("keys=").append(printPageKeys(io, pageAddr)).append(' ');
+
+ b.append(']');
+
+ return b.toString();
+ }
+
+ /**
+ * @param io IO.
+ * @param pageAddr Page address.
+ * @return Keys as String.
+ * @throws IgniteCheckedException If failed.
+ */
+ private String printPageKeys(BPlusIO<L> io, long pageAddr) throws IgniteCheckedException {
+ int cnt = io.getCount(pageAddr);
+
+ StringBuilder b = new StringBuilder();
+
+ b.append('[');
+
+ for (int i = 0; i < cnt; i++) {
+ if (i != 0)
+ b.append(',');
+
+ b.append(io.isLeaf() || canGetRowFromInner ? getRow(io, pageAddr, i) : io.getLookupRow(this, pageAddr, i));
+ }
+
+ b.append(']');
+
+ return b.toString();
+ }
+
+ /**
+ * @param x Long.
+ * @return String.
+ */
+ private static String formatPageId(long x) {
+ return U.hexLong(x);
+ }
+
+ /**
+ * @param idx Index after binary search, which can be negative.
+ * @return Always positive index.
+ */
+ private static int fix(int idx) {
+ assert checkIndex(idx): idx;
+
+ if (idx < 0)
+ idx = -idx - 1;
+
+ return idx;
+ }
+
+ /**
+ * @param idx Index.
+ * @return {@code true} If correct.
+ */
+ private static boolean checkIndex(int idx) {
+ return idx > -Short.MAX_VALUE && idx < Short.MAX_VALUE;
+ }
+
+ /**
+ * Check if interrupted.
+ * @throws IgniteInterruptedCheckedException If interrupted.
+ */
+ private static void checkInterrupted() throws IgniteInterruptedCheckedException {
+ // We should not interrupt operations in the middle, because otherwise we'll end up in inconsistent state.
+ // Because of that we do not check for Thread.interrupted()
+ if (interrupted)
+ throw new IgniteInterruptedCheckedException("Interrupted.");
+ }
+
+ /**
+ * Interrupt all operations on all threads and all indexes.
+ */
+ @SuppressWarnings("unused")
+ public static void interruptAll() {
+ interrupted = true;
+ }
+
+ /**
+ * @param row Lookup row.
+ * @return Removed row.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public final T remove(L row) throws IgniteCheckedException {
+ return doRemove(row, true);
+ }
+
+ /**
+ * @param row Lookup row.
+ * @throws IgniteCheckedException If failed.
+ * @return {@code True} if removed row.
+ */
+ public final boolean removex(L row) throws IgniteCheckedException {
+ Boolean res = (Boolean)doRemove(row, false);
+
+ return res != null ? res : false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void invoke(L row, Object z, InvokeClosure<T> c) throws IgniteCheckedException {
+ checkDestroyed();
+
+ Invoke x = new Invoke(row, z, c);
+
+ try {
+ for (;;) {
+ x.init();
+
+ Result res = invokeDown(x, x.rootId, 0L, 0L, x.rootLvl);
+
+ switch (res) {
+ case RETRY:
+ case RETRY_ROOT:
+ checkInterrupted();
+
+ continue;
+
+ default:
+ if (!x.isFinished()) {
+ res = x.tryFinish();
+
+ if (res == RETRY || res == RETRY_ROOT) {
+ checkInterrupted();
+
+ continue;
+ }
+
+ assert x.isFinished(): res;
+ }
+
+ return;
+ }
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on search row: " + row, e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on search row: " + row, e);
+ }
+ finally {
+ x.releaseAll();
+ checkDestroyed();
+ }
+ }
+
+ /**
+ * @param x Invoke operation.
+ * @param pageId Page ID.
+ * @param backId Expected backward page ID if we are going to the right.
+ * @param fwdId Expected forward page ID.
+ * @param lvl Level.
+ * @return Result code.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Result invokeDown(final Invoke x, final long pageId, final long backId, final long fwdId, final int lvl)
+ throws IgniteCheckedException {
+ assert lvl >= 0 : lvl;
+
+ if (x.isTail(pageId, lvl))
+ return FOUND; // We've already locked this page, so return that we are ok.
+
+ long page = acquirePage(pageId);
+
+ try {
+ for (;;) {
+ // Init args.
+ x.pageId(pageId);
+ x.fwdId(fwdId);
+ x.backId(backId);
+
+ Result res = read(pageId, page, search, x, lvl, RETRY);
+
+ switch (res) {
+ case GO_DOWN_X:
+ assert backId != 0;
+ assert x.backId == 0; // We did not setup it yet.
+
+ x.backId(pageId); // Dirty hack to setup a check inside of askNeighbor.
+
+ // We need to get backId here for our child page, it must be the last child of our back.
+ res = askNeighbor(backId, x, true);
+
+ if (res != FOUND)
+ return res; // Retry.
+
+ assert x.backId != pageId; // It must be updated in askNeighbor.
+
+ // Intentional fallthrough.
+ case GO_DOWN:
+ res = x.tryReplaceInner(pageId, page, fwdId, lvl);
+
+ if (res != RETRY)
+ res = invokeDown(x, x.pageId, x.backId, x.fwdId, lvl - 1);
+
+ if (res == RETRY_ROOT || x.isFinished())
+ return res;
+
+ if (res == RETRY) {
+ checkInterrupted();
+
+ continue;
+ }
+
+ // Unfinished Put does insertion on the same level.
+ if (x.isPut())
+ continue;
+
+ assert x.isRemove(); // Guarded by isFinished.
+
+ res = x.finishOrLockTail(pageId, page, backId, fwdId, lvl);
+
+ return res;
+
+ case NOT_FOUND:
+ if (lvl == 0)
+ x.invokeClosure();
+
+ return x.onNotFound(pageId, page, fwdId, lvl);
+
+ case FOUND:
+ if (lvl == 0)
+ x.invokeClosure();
+
+ return x.onFound(pageId, page, backId, fwdId, lvl);
+
+ default:
+ return res;
+ }
+ }
+ }
+ finally {
+ x.levelExit();
+
+ if (x.canRelease(pageId, lvl))
+ releasePage(pageId, page);
+ }
+ }
+
+
+ /**
+ * @param row Lookup row.
+ * @param needOld {@code True} if need return removed row.
+ * @return Removed row.
+ * @throws IgniteCheckedException If failed.
+ */
+ private T doRemove(L row, boolean needOld) throws IgniteCheckedException {
+ checkDestroyed();
+
+ Remove r = new Remove(row, needOld);
+
+ try {
+ for (;;) {
+ r.init();
+
+ Result res = removeDown(r, r.rootId, 0L, 0L, r.rootLvl);
+
+ switch (res) {
+ case RETRY:
+ case RETRY_ROOT:
+ checkInterrupted();
+
+ continue;
+
+ default:
+ if (!r.isFinished()) {
+ res = r.finishTail();
+
+ // If not found, then the tree grew beyond our call stack -> retry from the actual root.
+ if (res == RETRY || res == NOT_FOUND) {
+ assert r.checkTailLevel(getRootLevel()) : "tail=" + r.tail + ", res=" + res;
+
+ checkInterrupted();
+
+ continue;
+ }
+
+ assert res == FOUND: res;
+ }
+
+ assert r.isFinished();
+
+ return r.rmvd;
+ }
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on search row: " + row, e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on search row: " + row, e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on search row: " + row, e);
+ }
+ finally {
+ r.releaseAll();
+ checkDestroyed();
+ }
+ }
+
+ /**
+ * @param r Remove operation.
+ * @param pageId Page ID.
+ * @param backId Expected backward page ID if we are going to the right.
+ * @param fwdId Expected forward page ID.
+ * @param lvl Level.
+ * @return Result code.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Result removeDown(final Remove r, final long pageId, final long backId, final long fwdId, final int lvl)
+ throws IgniteCheckedException {
+ assert lvl >= 0 : lvl;
+
+ if (r.isTail(pageId, lvl))
+ return FOUND; // We've already locked this page, so return that we are ok.
+
+ long page = acquirePage(pageId);
+
+ try {
+ for (;;) {
+ // Init args.
+ r.pageId = pageId;
+ r.fwdId = fwdId;
+ r.backId = backId;
+
+ Result res = read(pageId, page, search, r, lvl, RETRY);
+
+ switch (res) {
+ case GO_DOWN_X:
+ assert backId != 0;
+ assert r.backId == 0; // We did not setup it yet.
+
+ r.backId = pageId; // Dirty hack to setup a check inside of askNeighbor.
+
+ // We need to get backId here for our child page, it must be the last child of our back.
+ res = askNeighbor(backId, r, true);
+
+ if (res != FOUND)
+ return res; // Retry.
+
+ assert r.backId != pageId; // It must be updated in askNeighbor.
+
+ // Intentional fallthrough.
+ case GO_DOWN:
+ res = removeDown(r, r.pageId, r.backId, r.fwdId, lvl - 1);
+
+ if (res == RETRY) {
+ checkInterrupted();
+
+ continue;
+ }
+
+ if (res == RETRY_ROOT || r.isFinished())
+ return res;
+
+ res = r.finishOrLockTail(pageId, page, backId, fwdId, lvl);
+
+ return res;
+
+ case NOT_FOUND:
+ // We are at the bottom.
+ assert lvl == 0 : lvl;
+
+ r.finish();
+
+ return res;
+
+ case FOUND:
+ return r.tryRemoveFromLeaf(pageId, page, backId, fwdId, lvl);
+
+ default:
+ return res;
+ }
+ }
+ }
+ finally {
+ r.page = 0L;
+
+ if (r.canRelease(pageId, lvl))
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param cnt Count.
+ * @param cap Capacity.
+ * @return {@code true} If may merge.
+ */
+ private boolean mayMerge(int cnt, int cap) {
+ int minCnt = (int)(minFill * cap);
+
+ if (cnt <= minCnt) {
+ assert cnt == 0; // TODO remove
+
+ return true;
+ }
+
+ assert cnt > 0;
+
+ int maxCnt = (int)(maxFill * cap);
+
+ if (cnt > maxCnt)
+ return false;
+
+ assert false; // TODO remove
+
+ // Randomization is for smoothing worst case scenarios. Probability of merge attempt
+ // is proportional to free space in our page (discounted on fill factor).
+ return randomInt(maxCnt - minCnt) >= cnt - minCnt;
+ }
+
+ /**
+ * @return Root level.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final int rootLevel() throws IgniteCheckedException {
+ checkDestroyed();
+
+ return getRootLevel();
+ }
+
+ /**
+ * !!! For debug only! May produce wrong results on concurrent access.
+ *
+ * @return Size.
+ * @throws IgniteCheckedException If failed.
+ */
+ @Override public final long size() throws IgniteCheckedException {
+ checkDestroyed();
+
+ long pageId;
+
+ long metaPage = acquirePage(metaPageId);
+ try {
+ pageId = getFirstPageId(metaPageId, metaPage, 0); // Level 0 is always at the bottom.
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+
+ BPlusIO<L> io = null;
+
+ long cnt = 0;
+
+ while (pageId != 0) {
+ long curId = pageId;
+ long curPage = acquirePage(curId);
+ try {
+ long curAddr = readLock(curId, curPage); // No correctness guaranties.
+
+ try {
+ if (io == null) {
+ io = io(curAddr);
+
+ assert io.isLeaf();
+ }
+
+ cnt += io.getCount(curAddr);
+
+ pageId = io.getForward(curAddr);
+ }
+ finally {
+ readUnlock(curId, curPage, curAddr);
+ }
+ }
+ finally {
+ releasePage(curId, curPage);
+ }
+ }
+
+ checkDestroyed();
+
+ return cnt;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public final T put(T row) throws IgniteCheckedException {
+ return doPut(row, true);
+ }
+
+ /**
+ * @param row New value.
+ * @throws IgniteCheckedException If failed.
+ * @return {@code True} if replaced existing row.
+ */
+ public boolean putx(T row) throws IgniteCheckedException {
+ Boolean res = (Boolean)doPut(row, false);
+
+ return res != null ? res : false;
+ }
+
+ /**
+ * @param row New value.
+ * @param needOld {@code True} If need return old value.
+ * @return Old row.
+ * @throws IgniteCheckedException If failed.
+ */
+ private T doPut(T row, boolean needOld) throws IgniteCheckedException {
+ checkDestroyed();
+
+ Put p = new Put(row, needOld);
+
+ try {
+ for (;;) { // Go down with retries.
+ p.init();
+
+ Result res = putDown(p, p.rootId, 0L, p.rootLvl);
+
+ switch (res) {
+ case RETRY:
+ case RETRY_ROOT:
+ checkInterrupted();
+
+ continue;
+
+ case FOUND:
+ // We may need to insert split key into upper level here.
+ if (!p.isFinished()) {
+ // It must be impossible to have an insert higher than the current root,
+ // because we are making decision about creating new root while keeping
+ // write lock on current root, so it can't concurrently change.
+ assert p.btmLvl <= getRootLevel();
+
+ checkInterrupted();
+
+ continue;
+ }
+
+ return p.oldRow;
+
+ default:
+ throw new IllegalStateException("Result: " + res);
+ }
+ }
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteCheckedException("Runtime failure on row: " + row, e);
+ }
+ catch (RuntimeException e) {
+ throw new IgniteException("Runtime failure on row: " + row, e);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError("Assertion error on row: " + row, e);
+ }
+ finally {
+ checkDestroyed();
+ }
+ }
+
+ /**
+ * Destroys tree. This method is allowed to be invoked only when the tree is out of use (no concurrent operations
+ * are trying to read or update the tree after destroy beginning).
+ *
+ * @return Number of pages recycled from this tree. If the tree was destroyed by someone else concurrently returns
+ * {@code 0}, otherwise it should return at least {@code 2} (for meta page and root page), unless this tree is
+ * used as metadata storage, or {@code -1} if we don't have a reuse list and did not do recycling at all.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final long destroy() throws IgniteCheckedException {
+ return destroy(null);
+ }
+
+ /**
+ * Destroys tree. This method is allowed to be invoked only when the tree is out of use (no concurrent operations
+ * are trying to read or update the tree after destroy beginning).
+ *
+ * @param c Visitor closure. Visits only leaf pages.
+ * @return Number of pages recycled from this tree. If the tree was destroyed by someone else concurrently returns
+ * {@code 0}, otherwise it should return at least {@code 2} (for meta page and root page), unless this tree is
+ * used as metadata storage, or {@code -1} if we don't have a reuse list and did not do recycling at all.
+ * @throws IgniteCheckedException If failed.
+ */
+ public final long destroy(IgniteInClosure<L> c) throws IgniteCheckedException {
+ if (!markDestroyed())
+ return 0;
+
+ if (reuseList == null)
+ return -1;
+
+ DestroyBag bag = new DestroyBag();
+
+ long pagesCnt = 0;
+
+ long metaPage = acquirePage(metaPageId);
+ try {
+ long metaPageAddr = writeLock(metaPageId, metaPage); // No checks, we must be out of use.
+
+ assert metaPageAddr != 0L;
+
+ try {
+ for (long pageId : getFirstPageIds(metaPageAddr)) {
+ assert pageId != 0;
+
+ do {
+ final long pId = pageId;
+
+ long page = acquirePage(pId);
+
+ try {
+ long pageAddr = writeLock(pId, page); // No checks, we must be out of use.
+
+ try {
+ BPlusIO<L> io = io(pageAddr);
+
+ if (c != null && io.isLeaf())
+ io.visit(pageAddr, c);
+
+ long fwdPageId = io.getForward(pageAddr);
+
+ bag.addFreePage(recyclePage(pageId, page, pageAddr, null));
+ pagesCnt++;
+
+ pageId = fwdPageId;
+ }
+ finally {
+ writeUnlock(pId, page, pageAddr, true);
+ }
+ }
+ finally {
+ releasePage(pId, page);
+ }
+
+ if (bag.size() == 128) {
+ reuseList.addForRecycle(bag);
+
+ assert bag.isEmpty() : bag.size();
+ }
+ }
+ while (pageId != 0);
+ }
+
+ bag.addFreePage(recyclePage(metaPageId, metaPage, metaPageAddr, null));
+ pagesCnt++;
+ }
+ finally {
+ writeUnlock(metaPageId, metaPage, metaPageAddr, true);
+ }
+ }
+ finally {
+ releasePage(metaPageId, metaPage);
+ }
+
+ reuseList.addForRecycle(bag);
+
+ assert bag.size() == 0 : bag.size();
+
+ return pagesCnt;
+ }
+
+ /**
+ * @return {@code True} if state was changed.
+ */
+ private boolean markDestroyed() {
+ return destroyed.compareAndSet(false, true);
+ }
+
+ /**
+ * @param pageAddr Meta page address.
+ * @return First page IDs.
+ */
+ protected Iterable<Long> getFirstPageIds(long pageAddr) {
+ List<Long> res = new ArrayList<>();
+
+ BPlusMetaIO mio = BPlusMetaIO.VERSIONS.forPage(pageAddr);
+
+ for (int lvl = mio.getRootLevel(pageAddr); lvl >= 0; lvl--)
+ res.add(mio.getFirstPageId(pageAddr, lvl));
+
+ return res;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address
+ * @param io IO.
+ * @param fwdId Forward page ID.
+ * @param fwdBuf Forward buffer.
+ * @param idx Insertion index.
+ * @return {@code true} The middle index was shifted to the right.
+ * @throws IgniteCheckedException If failed.
+ */
+ private boolean splitPage(
+ long pageId, long page, long pageAddr, BPlusIO io, long fwdId, long fwdBuf, int idx
+ ) throws IgniteCheckedException {
+ int cnt = io.getCount(pageAddr);
+ int mid = cnt >>> 1;
+
+ boolean res = false;
+
+ if (idx > mid) { // If insertion is going to be to the forward page, keep more in the back page.
+ mid++;
+
+ res = true;
+ }
+
+ // Update forward page.
+ io.splitForwardPage(pageAddr, fwdId, fwdBuf, mid, cnt, pageSize());
+
+ // Update existing page.
+ io.splitExistingPage(pageAddr, mid, fwdId);
+
+ if (needWalDeltaRecord(pageId, page, null))
+ wal.log(new SplitExistingPageRecord(cacheId, pageId, mid, fwdId));
+
+ return res;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address
+ * @param walPlc Full page WAL record policy.
+ */
+ private void writeUnlockAndClose(long pageId, long page, long pageAddr, Boolean walPlc) {
+ try {
+ writeUnlock(pageId, page, pageAddr, walPlc, true);
+ }
+ finally {
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param pageId Inner page ID.
+ * @param g Get.
+ * @param back Get back (if {@code true}) or forward page (if {@code false}).
+ * @return Operation result.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Result askNeighbor(long pageId, Get g, boolean back) throws IgniteCheckedException {
+ return read(pageId, askNeighbor, g, back ? TRUE.ordinal() : FALSE.ordinal(), RETRY);
+ }
+
+ /**
+ * @param p Put.
+ * @param pageId Page ID.
+ * @param fwdId Expected forward page ID.
+ * @param lvl Level.
+ * @return Result code.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Result putDown(final Put p, final long pageId, final long fwdId, final int lvl)
+ throws IgniteCheckedException {
+ assert lvl >= 0 : lvl;
+
+ final long page = acquirePage(pageId);
+
+ try {
+ for (;;) {
+ // Init args.
+ p.pageId = pageId;
+ p.fwdId = fwdId;
+
+ Result res = read(pageId, page, search, p, lvl, RETRY);
+
+ switch (res) {
+ case GO_DOWN:
+ case GO_DOWN_X:
+ assert lvl > 0 : lvl;
+ assert p.pageId != pageId;
+ assert p.fwdId != fwdId || fwdId == 0;
+
+ res = p.tryReplaceInner(pageId, page, fwdId, lvl);
+
+ if (res != RETRY) // Go down recursively.
+ res = putDown(p, p.pageId, p.fwdId, lvl - 1);
+
+ if (res == RETRY_ROOT || p.isFinished())
+ return res;
+
+ if (res == RETRY)
+ checkInterrupted();
+
+ continue; // We have to insert split row to this level or it is a retry.
+
+ case FOUND: // Do replace.
+ assert lvl == 0 : "This replace can happen only at the bottom level.";
+
+ return p.tryReplace(pageId, page, fwdId, lvl);
+
+ case NOT_FOUND: // Do insert.
+ assert lvl == p.btmLvl : "must insert at the bottom level";
+ assert p.needReplaceInner == FALSE : p.needReplaceInner + " " + lvl;
+
+ return p.tryInsert(pageId, page, fwdId, lvl);
+
+ default:
+ return res;
+ }
+ }
+ }
+ finally {
+ if (p.canRelease(pageId, lvl))
+ releasePage(pageId, page);
+ }
+ }
+
+ /**
+ * @param io IO.
+ * @param pageAddr Page address.
+ * @param back Backward page.
+ * @return Page ID.
+ */
+ private long doAskNeighbor(BPlusIO<L> io, long pageAddr, boolean back) {
+ long res;
+
+ if (back) {
+ // Count can be 0 here if it is a routing page, in this case we have a single child.
+ int cnt = io.getCount(pageAddr);
+
+ // We need to do get the rightmost child: io.getRight(cnt - 1),
+ // here io.getLeft(cnt) is the same, but handles negative index if count is 0.
+ res = inner(io).getLeft(pageAddr, cnt);
+ }
+ else // Leftmost child.
+ res = inner(io).getLeft(pageAddr, 0);
+
+ assert res != 0 : "inner page with no route down: " + U.hexLong(PageIO.getPageId(pageAddr));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(BPlusTree.class, this);
+ }
+
+ /**
+ * Get operation.
+ */
+ private abstract class Get {
+ /** */
+ long rmvId;
+
+ /** Starting point root level. May be outdated. Must be modified only in {@link Get#init()}. */
+ int rootLvl;
+
+ /** Starting point root ID. May be outdated. Must be modified only in {@link Get#init()}. */
+ long rootId;
+
+ /** */
+ L row;
+
+ /** In/Out parameter: Page ID. */
+ long pageId;
+
+ /** In/Out parameter: expected forward page ID. */
+ long fwdId;
+
+ /** In/Out parameter: in case of right turn this field will contain backward page ID for the child. */
+ long backId;
+
+ /** */
+ int shift;
+
+ /** If this operation is a part of invoke. */
+ Invoke invoke;
+
+ /** Ignore row passed, find last row */
+ boolean findLast;
+
+ /**
+ * @param row Row.
+ * @param findLast find last row.
+ */
+ Get(L row, boolean findLast) {
+ assert findLast ^ row != null;
+
+ this.row = row;
+ this.findLast = findLast;
+ }
+
+ /**
+ * @param g Other operation to copy from.
+ */
+ final void copyFrom(Get g) {
+ rmvId = g.rmvId;
+ rootLvl = g.rootLvl;
+ pageId = g.pageId;
+ fwdId = g.fwdId;
+ backId = g.backId;
+ shift = g.shift;
+ findLast = g.findLast;
+ }
+
+ /**
+ * Initialize operation.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ final void init() throws IgniteCheckedException {
+ TreeMetaData meta0 = treeMeta();
+
+ assert meta0 != null;
+
+ restartFromRoot(meta0.rootId, meta0.rootLvl, globalRmvId.get());
+ }
+
+ /**
+ * @param rootId Root page ID.
+ * @param rootLvl Root level.
+ * @param rmvId Remove ID to be afraid of.
+ */
+ void restartFromRoot(long rootId, int rootLvl, long rmvId) {
+ this.rootId = rootId;
+ this.rootLvl = rootLvl;
+ this.rmvId = rmvId;
+ }
+
+ /**
+ * @param io IO.
+ * @param pageAddr Page address.
+ * @param idx Index of found entry.
+ * @param lvl Level.
+ * @return {@code true} If we need to stop.
+ * @throws IgniteCheckedException If failed.
+ */
+ boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
+ assert lvl >= 0;
+
+ return lvl == 0; // Stop if we are at the bottom.
+ }
+
+ /**
+ * @param io IO.
+ * @param pageAddr Page address.
+ * @param idx Insertion point.
+ * @param lvl Level.
+ * @return {@code true} If we need to stop.
+ * @throws IgniteCheckedException If failed.
+ */
+ boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
+ assert lvl >= 0;
+
+ return lvl == 0; // Stop if we are at the bottom.
+ }
+
+ /**
+ * @param pageId Page.
+ * @param lvl Level.
+ * @return {@code true} If we can release the given page.
+ */
+ boolean canRelease(long pageId, int lvl) {
+ return pageId != 0L;
+ }
+
+ /**
+ * @param backId Back page ID.
+ */
+ void backId(long backId) {
+ this.backId = backId;
+ }
+
+ /**
+ * @param pageId Page ID.
+ */
+ void pageId(long pageId) {
+ this.pageId = pageId;
+ }
+
+ /**
+ * @param fwdId Forward page ID.
+ */
+ void fwdId(long fwdId) {
+ this.fwdId = fwdId;
+ }
+
+ /**
+ * @return {@code true} If the operation is finished.
+ */
+ boolean isFinished() {
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Get a single entry.
+ */
+ private final class GetOne extends Get {
+ /** */
+ Object x;
+
+ /**
+ * @param row Row.
+ * @param x Implementation specific argument.
+ * @param findLast Ignore row passed, find last row
+ */
+ private GetOne(L row, Object x, boolean findLast) {
+ super(row, findLast);
+
+ this.x = x;
+ }
+
+ /** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
+ @Override boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
+ // Check if we are on an inner page and can't get row from it.
+ if (lvl != 0 && !canGetRowFromInner)
+ return false;
+
+ row = getRow(io, pageAddr, idx, x);
+
+ return true;
+ }
+ }
+
+ /**
+ * Get a cursor for range.
+ */
+ private final class GetCursor extends Get {
+ /** */
+ ForwardCursor cursor;
+
+ /**
+ * @param lower Lower bound.
+ * @param shift Shift.
+ * @param cursor Cursor.
+ */
+ GetCursor(L lower, int shift, ForwardCursor cursor) {
+ super(lower, false);
+
+ assert shift != 0; // Either handle range of equal rows or find a greater row after concurrent merge.
+
+ this.shift = shift;
+ this.cursor = cursor;
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
+ throw new IllegalStateException(); // Must never be called because we always have a shift.
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) throws IgniteCheckedException {
+ if (lvl != 0)
+ return false;
+
+ cursor.init(pageAddr, io, idx);
+
+ return true;
+ }
+ }
+
+ /**
+ * Put operation.
+ */
+ private final class Put extends Get {
+ /** Right child page ID for split row. */
+ long rightId;
+
+ /** Replaced row if any. */
+ T oldRow;
+
+ /**
+ * This page is kept locked after split until insert to the upper level will not be finished.
+ * It is needed because split row will be "in flight" and if we'll release tail, remove on
+ * split row may fail.
+ */
+ long tailId;
+
+ /** */
+ long tailPage;
+
+ /** */
+ long tailAddr;
+
+ /**
+ * Bottom level for insertion (insert can't go deeper). Will be incremented on split on each level.
+ */
+ short btmLvl;
+
+ /** */
+ Bool needReplaceInner = FALSE;
+
+ /** */
+ final boolean needOld;
+
+ /**
+ * @param row Row.
+ * @param needOld {@code True} If need return old value.
+ */
+ private Put(T row, boolean needOld) {
+ super(row, false);
+
+ this.needOld = needOld;
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean found(BPlusIO<L> io, long pageAddr, int idx, int lvl) {
+ if (lvl == 0) // Leaf: need to stop.
+ return true;
+
+ assert btmLvl == 0; // It can not be insert.
+
+ // If we can get full row from the inner page, we have to replace it with the new one. On the way down
+ // we can not miss inner key even in presence of concurrent operations because of `triangle` invariant +
+ // concurrent inner replace handling by retrying from root.
+ if (canGetRowFromInner && needReplaceInner == FALSE)
+ needReplaceInner = TRUE;
+
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean notFound(BPlusIO<L> io, long pageAddr, int idx, int lvl) {
+ assert btmLvl >= 0 : btmLvl;
+ assert lvl >= btmLvl : lvl;
+
+ return lvl == btmLvl;
+ }
+
+ /**
+ * @param tailId Tail page ID.
+ * @param tailPage Tail page pointer.
+ * @param tailPageAddr Tail page address
+ */
+ private void tail(long tailId, long tailPage, long tailPageAddr) {
+ assert (tailId == 0L) == (tailPage == 0L);
+ assert (tailPage == 0L) == (tailPageAddr == 0L);
+
+ if (this.tailPage != 0L)
+ writeUnlockAndClose(this.tailId, this.tailPage, this.tailAddr, null);
+
+ this.tailId = tailId;
+ this.tailPage = tailPage;
+ this.tailAddr = tailPageAddr;
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean canRelease(long pageId, int lvl) {
+ return pageId != 0L && tailId != pageId;
+ }
+
+ /**
+ * Finish put.
+ */
+ private void finish() {
+ row = null;
+ rightId = 0;
+
+ tail(0L, 0L, 0L);
+ }
+
+ /** {@inheritDoc} */
+ @Override boolean isFinished() {
+ return row == null;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param idx Index.
+ * @param lvl Level.
+ * @return Move up row.
+ * @throws IgniteCheckedException If failed.
+ */
+ private L insert(long pageId, long page, long pageAddr, BPlusIO<L> io, int idx, int lvl)
+ throws IgniteCheckedException {
<TRUNCATED>
[23/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsTransactionsHangTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsTransactionsHangTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsTransactionsHangTest.java
new file mode 100644
index 0000000..d4dfdec
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsTransactionsHangTest.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.Random;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteTxManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.communication.tcp.TcpCommunicationSpi;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.jsr166.LongAdder8;
+
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Checks that transactions don't hang during checkpoint creation.
+ */
+public class IgnitePdsTransactionsHangTest extends GridCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Page cache size. */
+ private static final int PAGE_CACHE_SIZE = 512;
+
+ /** Page size. */
+ private static final Integer PAGE_SIZE = 16;
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "IgnitePdsTransactionsHangTest";
+
+ /** Number of insert threads. */
+ public static final int THREADS_CNT = 32;
+
+ /** Warm up period, seconds. */
+ public static final int WARM_UP_PERIOD = 20;
+
+ /** Duration. */
+ public static final int DURATION = 180;
+
+ /** Maximum count of inserted keys. */
+ public static final int MAX_KEY_COUNT = 500_000;
+
+ /** Checkpoint frequency. */
+ public static final long CHECKPOINT_FREQUENCY = 20_000;
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return Long.MAX_VALUE;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+ discoSpi.setIpFinder(IP_FINDER);
+ cfg.setDiscoverySpi(discoSpi);
+
+ BinaryConfiguration binaryCfg = new BinaryConfiguration();
+ binaryCfg.setCompactFooter(false);
+ cfg.setBinaryConfiguration(binaryCfg);
+
+ cfg.setPeerClassLoadingEnabled(true);
+
+ TcpCommunicationSpi tcpCommSpi = new TcpCommunicationSpi();
+
+ tcpCommSpi.setSharedMemoryPort(-1);
+ cfg.setCommunicationSpi(tcpCommSpi);
+
+ TransactionConfiguration txCfg = new TransactionConfiguration();
+
+ txCfg.setDefaultTxIsolation(TransactionIsolation.READ_COMMITTED);
+
+ cfg.setTransactionConfiguration(txCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalHistorySize(1)
+ .setCheckpointingFrequency(CHECKPOINT_FREQUENCY)
+ );
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(PAGE_CACHE_SIZE * 1024 * 1024);
+ memPlcCfg.setMaxSize(PAGE_CACHE_SIZE * 1024 * 1024);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ memCfg.setPageSize(PAGE_SIZE * 1024);
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ return cfg;
+ }
+
+ /**
+ * Creates cache configuration.
+ *
+ * @return Cache configuration.
+ * */
+ private CacheConfiguration getCacheConfiguration() {
+ CacheConfiguration ccfg = new CacheConfiguration();
+
+ ccfg.setName(CACHE_NAME);
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4));
+ ccfg.setReadFromBackup(true);
+
+ ccfg.setCacheMode(CacheMode.PARTITIONED);
+
+ return ccfg;
+ }
+
+ /**
+ * Copied from customers benchmark.
+ *
+ * @throws Exception If failed.
+ * */
+ public void testTransactionsDontHang() throws Exception {
+ try {
+ final Ignite g = startGrids(2);
+
+ g.getOrCreateCache(getCacheConfiguration());
+
+ ExecutorService threadPool = Executors.newFixedThreadPool(THREADS_CNT);
+ final CyclicBarrier cyclicBarrier = new CyclicBarrier(THREADS_CNT);
+
+ final AtomicBoolean interrupt = new AtomicBoolean(false);
+ final LongAdder8 operationCnt = new LongAdder8();
+
+ final IgniteCache<Long, TestEntity> cache = g.cache(CACHE_NAME);
+
+ for (int i = 0; i < THREADS_CNT; i++) {
+ threadPool.submit(new Runnable() {
+ @Override public void run() {
+ try {
+ ThreadLocalRandom locRandom = ThreadLocalRandom.current();
+
+ cyclicBarrier.await();
+
+ while (!interrupt.get()) {
+ long randomKey = locRandom.nextLong(MAX_KEY_COUNT);
+
+ TestEntity entity = TestEntity.newTestEntity(locRandom);
+
+ try (Transaction tx = g.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(randomKey, entity);
+
+ tx.commit();
+ }
+
+ operationCnt.increment();
+ }
+ }
+ catch (Throwable e) {
+ System.out.println(e.toString());
+
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ long stopTime = System.currentTimeMillis() + DURATION * 1000;
+ long totalOperations = 0;
+ int periods = 0;
+ long max = Long.MIN_VALUE, min = Long.MAX_VALUE;
+
+ while (System.currentTimeMillis() < stopTime) {
+ U.sleep(1000);
+
+ long sum = operationCnt.sumThenReset();
+ periods++;
+
+ if (periods > WARM_UP_PERIOD) {
+ totalOperations += sum;
+
+ max = Math.max(max, sum);
+ min = Math.min(min, sum);
+
+ System.out.println("Operation count: " + sum + " min=" + min + " max=" + max + " avg=" + totalOperations / (periods - WARM_UP_PERIOD));
+ }
+ }
+
+ interrupt.set(true);
+
+ threadPool.shutdown();
+ System.out.println("Test complete");
+
+ threadPool.awaitTermination(getTestTimeout(), TimeUnit.MILLISECONDS);
+
+ IgniteTxManager tm = internalCache(cache).context().tm();
+
+ assertEquals("There are still active transactions", 0, tm.activeTransactions().size());
+ } finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * Entity for test.
+ * */
+ public static class TestEntity {
+ /** String value. */
+ private String strVal;
+
+ /** Long value. */
+ private Long longVal;
+
+ /** Int value. */
+ private int intVal;
+
+ /**
+ * @param strVal String value.
+ */
+ public void setStrVal(String strVal) {
+ this.strVal = strVal;
+ }
+
+ /**
+ * @param longVal Long value.
+ */
+ public void setLongVal(Long longVal) {
+ this.longVal = longVal;
+ }
+
+ /**
+ * @param intVal Integer value.
+ */
+ public void setIntVal(int intVal) {
+ this.intVal = intVal;
+ }
+
+ /**
+ * Creates test entity with random values.
+ *
+ * @param random Random seq generator.
+ * @return new test entity
+ * */
+ private static TestEntity newTestEntity(Random random) {
+ TestEntity entity = new TestEntity();
+
+ entity.setLongVal((long) random.nextInt(1_000));
+ entity.setIntVal(random.nextInt(1_000));
+ entity.setStrVal("test" + random.nextInt(1_000));
+
+ return entity;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java
new file mode 100644
index 0000000..b512a64
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsWholeClusterRestartTest.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.checkpoint.noop.NoopCheckpointSpi;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsWholeClusterRestartTest extends GridCommonAbstractTest {
+ /** */
+ private static final int GRID_CNT = 5;
+
+ /** */
+ private static final int ENTRIES_COUNT = 1_000;
+
+ /** */
+ public static final String CACHE_NAME = "cache1";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg1 = new CacheConfiguration();
+
+ ccfg1.setName(CACHE_NAME);
+ ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg1.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg1.setBackups(2);
+
+ cfg.setLateAffinityAssignment(false);
+ cfg.setActiveOnStart(false);
+
+ // To avoid hostname lookup on start.
+ cfg.setCheckpointSpi(new NoopCheckpointSpi());
+
+ cfg.setCacheConfiguration(ccfg1);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setConsistentId(gridName);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRestarts() throws Exception {
+ startGrids(GRID_CNT);
+
+ ignite(0).active(true);
+
+ awaitPartitionMapExchange();
+
+ try (IgniteDataStreamer<Object, Object> ds = ignite(0).dataStreamer(CACHE_NAME)) {
+ for (int i = 0; i < ENTRIES_COUNT; i++)
+ ds.addData(i, i);
+ }
+
+ stopAllGrids();
+
+ List<Integer> idxs = new ArrayList<>();
+
+ for (int i = 0; i < GRID_CNT; i++)
+ idxs.add(i);
+
+ for (int r = 0; r < 10; r++) {
+ Collections.shuffle(idxs);
+
+ info("Will start in the following order: " + idxs);
+
+ for (Integer idx : idxs)
+ startGrid(idx);
+
+ try {
+ ignite(0).active(true);
+
+ for (int g = 0; g < GRID_CNT; g++) {
+ Ignite ig = ignite(g);
+
+ for (int k = 0; k < ENTRIES_COUNT; k++)
+ assertEquals("Failed to read [g=" + g + ", part=" + ig.affinity(CACHE_NAME).partition(k) +
+ ", nodes=" + ig.affinity(CACHE_NAME).mapKeyToPrimaryAndBackups(k) + ']',
+ k, ig.cache(CACHE_NAME).get(k));
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheIntegrationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheIntegrationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheIntegrationTest.java
new file mode 100644
index 0000000..c68f7e7
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCacheIntegrationTest.java
@@ -0,0 +1,285 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsCacheIntegrationTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int GRID_CNT = 3;
+
+ /** */
+ private static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+ ccfg.setIndexedTypes(Integer.class, DbValue.class);
+
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration bCfg = new BinaryConfiguration();
+
+ bCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(bCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPutGetSimple() throws Exception {
+ startGrids(GRID_CNT);
+
+ try {
+ IgniteEx ig = grid(0);
+
+ checkPutGetSql(ig, true);
+ }
+ finally {
+ stopAllGrids();
+ }
+
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+ info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
+
+ startGrids(GRID_CNT);
+
+ try {
+ IgniteEx ig = grid(0);
+
+ checkPutGetSql(ig, false);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPutMultithreaded() throws Exception {
+ startGrids(4);
+
+ try {
+ final IgniteEx grid = grid(0);
+
+ GridTestUtils.runMultiThreaded(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ for (int i = 0; i < 1000; i++)
+ grid.cache(CACHE_NAME).put(i, i);
+
+ return null;
+ }
+ }, 8, "updater");
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ig Ignite instance.
+ * @param write Write flag.
+ */
+ private void checkPutGetSql(Ignite ig, boolean write) {
+ IgniteCache<Integer, DbValue> cache = ig.cache(CACHE_NAME);
+
+ int entryCnt = 50_000;
+
+ if (write) {
+ try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(CACHE_NAME)) {
+ streamer.allowOverwrite(true);
+
+ for (int i = 0; i < entryCnt; i++)
+ streamer.addData(i, new DbValue(i, "value-" + i, i));
+ }
+ }
+
+ for (int i = 0; i < GRID_CNT; i++) {
+ IgniteEx ignite = grid(i);
+
+ GridCacheAdapter<Object, Object> cache0 = ignite.context().cache().internalCache(CACHE_NAME);
+
+ for (int k = 0; k < entryCnt; k++)
+ assertNull(cache0.peekEx(i));
+
+ assertEquals(entryCnt, ignite.cache(CACHE_NAME).size());
+ }
+
+ for (int i = 0; i < entryCnt; i++)
+ assertEquals("i = " + i, new DbValue(i, "value-" + i, i), cache.get(i));
+
+ List<List<?>> res = cache.query(new SqlFieldsQuery("select ival from dbvalue where ival < ? order by ival asc")
+ .setArgs(10_000)).getAll();
+
+ assertEquals(10_000, res.size());
+
+ for (int i = 0; i < 10_000; i++) {
+ assertEquals(1, res.get(i).size());
+ assertEquals(i, res.get(i).get(0));
+ }
+
+ assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
+ assertEquals(5000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 10000"))
+ .getAll().size());
+
+ for (int i = 0; i < 10_000; i++)
+ assertEquals(new DbValue(i, "value-" + i, i), cache.get(i));
+ }
+
+ /**
+ *
+ */
+ private static class DbValue implements Serializable {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String sVal;
+
+ /** */
+ @QuerySqlField
+ private long lVal;
+
+ /**
+ * @param iVal Integer value.
+ * @param sVal String value.
+ * @param lVal Long value.
+ */
+ private DbValue(int iVal, String sVal, long lVal) {
+ this.iVal = iVal;
+ this.sVal = sVal;
+ this.lVal = lVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ DbValue dbVal = (DbValue)o;
+
+ return iVal == dbVal.iVal && lVal == dbVal.lVal &&
+ !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
+ res = 31 * res + (int)(lVal ^ (lVal >>> 32));
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(DbValue.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
new file mode 100644
index 0000000..1026d4e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsEvictionTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test for page evictions.
+ */
+public class IgnitePdsEvictionTest extends GridCommonAbstractTest {
+ /** */
+ private static final int NUMBER_OF_SEGMENTS = 64;
+
+ /** */
+ private static final int PAGE_SIZE = 1024;
+
+ /** */
+ private static final long CHUNK_SIZE = 1024 * 1024;
+
+ /** */
+ private static final long MEMORY_LIMIT = 10 * CHUNK_SIZE;
+
+ /** */
+ private static final int PAGES_NUM = 128_000;
+
+ /** Cache name. */
+ private final String cacheName = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+ cfg.setMemoryConfiguration(createDbConfig());
+
+ cfg.setCacheConfiguration(new CacheConfiguration<>(cacheName));
+
+ return cfg;
+ }
+
+ /**
+ * @return DB config.
+ */
+ private MemoryConfiguration createDbConfig() {
+ final MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+ memPlcCfg.setInitialSize(MEMORY_LIMIT);
+ memPlcCfg.setMaxSize(MEMORY_LIMIT);
+ memPlcCfg.setName("dfltMemPlc");
+
+ memCfg.setPageSize(PAGE_SIZE);
+ memCfg.setConcurrencyLevel(NUMBER_OF_SEGMENTS);
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ return memCfg;
+ }
+
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testPageEviction() throws Exception {
+ final IgniteEx ig = startGrid(0);
+
+ final PageMemory memory = getMemory(ig);
+
+ writeData(ig, memory, CU.cacheId(cacheName));
+ }
+
+ /**
+ * @param memory Page memory.
+ * @param cacheId Cache id.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void writeData(final IgniteEx ignite, final PageMemory memory, final int cacheId) throws Exception {
+ final int size = PAGES_NUM;
+
+ final List<FullPageId> pageIds = new ArrayList<>(size);
+
+ IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
+
+ // Allocate.
+ for (int i = 0; i < size; i++) {
+ db.checkpointReadLock();
+ try {
+ final FullPageId fullId = new FullPageId(memory.allocatePage(cacheId, i % 256, PageMemory.FLAG_DATA),
+ cacheId);
+
+ pageIds.add(fullId);
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ System.out.println("Allocated pages: " + pageIds.size());
+
+ // Write data. (Causes evictions.)
+ final int part = PAGES_NUM / NUMBER_OF_SEGMENTS;
+
+ final Collection<IgniteInternalFuture> futs = new ArrayList<>();
+
+ for (int i = 0; i < PAGES_NUM; i += part)
+ futs.add(runWriteInThread(ignite, i, i + part, memory, pageIds));
+
+ for (final IgniteInternalFuture fut : futs)
+ fut.get();
+
+ System.out.println("Wrote pages: " + pageIds.size());
+
+ // Read data. (Causes evictions.)
+ futs.clear();
+
+ for (int i = 0; i < PAGES_NUM; i += part)
+ futs.add(runReadInThread(ignite, i, i + part, memory, pageIds));
+
+ for (final IgniteInternalFuture fut : futs)
+ fut.get();
+
+ System.out.println("Read pages: " + pageIds.size());
+ }
+
+ /**
+ * @param start Start index.
+ * @param end End index.
+ * @param memory PageMemory.
+ * @param pageIds Allocated pages.
+ * @return Future.
+ * @throws Exception If fail.
+ */
+ private IgniteInternalFuture runWriteInThread(
+ final IgniteEx ignite,
+ final int start,
+ final int end,
+ final PageMemory memory,
+ final List<FullPageId> pageIds
+ ) throws Exception {
+
+ return GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
+
+ for (int i = start; i < end; i++) {
+ db.checkpointReadLock();
+
+ try {
+ FullPageId fullId = pageIds.get(i);
+
+ long page = memory.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ final long pageAddr = memory.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ try {
+ PageIO.setPageId(pageAddr, fullId.pageId());
+
+ PageUtils.putLong(pageAddr, PageIO.COMMON_HEADER_END, i * 2);
+ }
+ finally {
+ memory.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ memory.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * @param start Start index.
+ * @param end End index.
+ * @param memory PageMemory.
+ * @param pageIds Allocated pages.
+ * @return Future.
+ * @throws Exception If fail.
+ */
+ private IgniteInternalFuture runReadInThread(final IgniteEx ignite, final int start, final int end,
+ final PageMemory memory,
+ final List<FullPageId> pageIds) throws Exception {
+ return GridTestUtils.runAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgniteCacheDatabaseSharedManager db = ignite.context().cache().context().database();
+
+ for (int i = start; i < end; i++) {
+ db.checkpointReadLock();
+
+ try {
+ final FullPageId fullId = pageIds.get(i);
+
+ long page = memory.acquirePage(fullId.cacheId(), fullId.pageId());
+ try {
+ final long pageAddr = memory.readLock(fullId.cacheId(), fullId.pageId(), page);
+
+ try {
+ assertEquals(i * 2, PageUtils.getLong(pageAddr, PageIO.COMMON_HEADER_END));
+ }
+ finally {
+ memory.readUnlock(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ finally {
+ memory.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ return null;
+ }
+ });
+ }
+
+ /**
+ * @param ig Ignite instance.
+ * @return Memory and store.
+ * @throws Exception If failed to initialize the store.
+ */
+ private PageMemory getMemory(IgniteEx ig) throws Exception {
+ final GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
+
+ final IgniteCacheDatabaseSharedManager db = sharedCtx.database();
+
+ return db.memoryPolicy(null).pageMemory();
+ }
+
+ /**
+ * @throws IgniteCheckedException If fail.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsNoActualWalHistoryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsNoActualWalHistoryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsNoActualWalHistoryTest.java
new file mode 100644
index 0000000..9631686
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsNoActualWalHistoryTest.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgnitePdsNoActualWalHistoryTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setPageSize(4 * 1024);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalSegmentSize(4 * 1024 * 1024)
+ .setWalHistorySize(2)
+ .setWalSegments(10)
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ binCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalBig() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ IgniteCache<Object, Object> cache = ignite.cache(CACHE_NAME);
+
+ Random rnd = new Random();
+
+ Map<Integer, IndexedObject> map = new HashMap<>();
+
+ for (int i = 0; i < 40_000; i++) {
+ if (i % 1000 == 0)
+ X.println(" >> " + i);
+
+ int k = rnd.nextInt(300_000);
+ IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
+
+ cache.put(k, v);
+ map.put(k, v);
+ }
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ignite.context().cache().context()
+ .database();
+
+ // Create many checkpoints to clean up the history.
+ dbMgr.wakeupForCheckpoint("test").get();
+ dbMgr.wakeupForCheckpoint("test").get();
+ dbMgr.wakeupForCheckpoint("test").get();
+ dbMgr.wakeupForCheckpoint("test").get();
+ dbMgr.wakeupForCheckpoint("test").get();
+ dbMgr.wakeupForCheckpoint("test").get();
+ dbMgr.wakeupForCheckpoint("test").get();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ for (int i = 0; i < 50; i++) {
+ int k = rnd.nextInt(300_000);
+ IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
+
+ cache.put(k, v);
+ map.put(k, v);
+ }
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ cache = ignite.cache(CACHE_NAME);
+
+ // Check.
+ for (Integer k : map.keySet())
+ assertEquals(map.get(k), cache.get(k));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class IndexedObject {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ private byte[] payload = new byte[1024];
+
+ /**
+ * @param iVal Integer value.
+ */
+ private IndexedObject(int iVal) {
+ this.iVal = iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof IndexedObject))
+ return false;
+
+ IndexedObject that = (IndexedObject)o;
+
+ return iVal == that.iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IndexedObject.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private enum EnumVal {
+ /** */
+ VAL1,
+
+ /** */
+ VAL2,
+
+ /** */
+ VAL3
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/ignitePdsCheckpointSimulationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/ignitePdsCheckpointSimulationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/ignitePdsCheckpointSimulationTest.java
new file mode 100644
index 0000000..840042e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/ignitePdsCheckpointSimulationTest.java
@@ -0,0 +1,1000 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.nio.ByteOrder;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
+import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheOperation;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class ignitePdsCheckpointSimulationTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int TOTAL_PAGES = 1000;
+
+ /** */
+ private static final boolean VERBOSE = false;
+
+ /** Cache name. */
+ private final String cacheName = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration ccfg = new CacheConfiguration(cacheName);
+
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setCheckpointingFrequency(500)
+ .setWalMode(WALMode.LOG_ONLY)
+ .setAlwaysWriteFullPages(true)
+ );
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testCheckpointSimulationMultiThreaded() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
+
+ IgnitePageStoreManager pageStore = shared.pageStore();
+
+ U.sleep(1_000);
+
+ // Disable integrated checkpoint thread.
+ dbMgr.enableCheckpoints(false).get();
+
+ // Must put something in partition 0 in order to initialize meta page.
+ // Otherwise we will violate page store integrity rules.
+ ig.cache(cacheName).put(0, 0);
+
+ PageMemory mem = shared.database().memoryPolicy(null).pageMemory();
+
+ IgniteBiTuple<Map<FullPageId, Integer>, WALPointer> res;
+
+ try {
+ res = runCheckpointing(ig, (PageMemoryImpl)mem, pageStore, shared.wal(),
+ shared.cache().cache(cacheName).context().cacheId());
+ }
+ catch (Throwable th) {
+ log().error("Error while running checkpointing", th);
+
+ throw th;
+ }
+ finally {
+ dbMgr.enableCheckpoints(true).get();
+
+ stopAllGrids(false);
+ }
+
+ ig = startGrid(0);
+
+ shared = ig.context().cache().context();
+
+ dbMgr = (GridCacheDatabaseSharedManager)shared.database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ mem = shared.database().memoryPolicy(null).pageMemory();
+
+ verifyReads(res.get1(), mem, res.get2(), shared.wal());
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testGetForInitialWrite() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
+
+ int cacheId = shared.cache().cache(cacheName).context().cacheId();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
+
+ // Disable integrated checkpoint thread.
+ dbMgr.enableCheckpoints(false);
+
+ PageMemory mem = shared.database().memoryPolicy(null).pageMemory();
+
+ IgniteWriteAheadLogManager wal = shared.wal();
+
+ WALPointer start = wal.log(new CheckpointRecord(null, false));
+
+ final FullPageId[] initWrites = new FullPageId[10];
+
+ ig.context().cache().context().database().checkpointReadLock();
+
+ try {
+ for (int i = 0; i < initWrites.length; i++)
+ initWrites[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+
+ // Check getForInitialWrite methods.
+ for (FullPageId fullId : initWrites) {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+ try {
+ long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ try {
+ DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
+
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
+ PageUtils.putByte(pageAddr, i, (byte)0xAB);
+ }
+ finally {
+ mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+
+ wal.fsync(null);
+ }
+ finally {
+ ig.context().cache().context().database().checkpointReadUnlock();
+ stopAllGrids(false);
+ }
+
+ ig = startGrid(0);
+
+ shared = ig.context().cache().context();
+
+ dbMgr = (GridCacheDatabaseSharedManager)shared.database();
+
+ dbMgr.enableCheckpoints(false);
+
+ wal = shared.wal();
+
+ try (WALIterator it = wal.replay(start)) {
+ it.nextX();
+
+ for (FullPageId initialWrite : initWrites) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+ assertTrue(String.valueOf(tup.get2()), tup.get2() instanceof PageSnapshot);
+
+ PageSnapshot snap = (PageSnapshot)tup.get2();
+
+ FullPageId actual = snap.fullPageId();
+
+ //there are extra tracking pages, skip them
+ if (TrackingPageIO.VERSIONS.latest().trackingPageFor(actual.pageId(), mem.pageSize()) == actual.pageId()) {
+ tup = it.nextX();
+
+ assertTrue(tup.get2() instanceof PageSnapshot);
+
+ actual = ((PageSnapshot)tup.get2()).fullPageId();
+ }
+
+ assertEquals(initialWrite, actual);
+ }
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testDataWalEntries() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
+ GridCacheContext<Object, Object> cctx = sharedCtx.cache().cache(cacheName).context();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+ IgniteWriteAheadLogManager wal = sharedCtx.wal();
+
+ assertTrue(wal.isAlwaysWriteFullPages());
+
+ db.enableCheckpoints(false);
+
+ final int cnt = 10;
+
+ List<DataEntry> entries = new ArrayList<>(cnt);
+
+ for (int i = 0; i < cnt; i++) {
+ GridCacheOperation op = i % 2 == 0 ? GridCacheOperation.UPDATE : GridCacheOperation.DELETE;
+
+ KeyCacheObject key = cctx.toCacheKeyObject(i);
+
+ CacheObject val = null;
+
+ if (op != GridCacheOperation.DELETE)
+ val = cctx.toCacheObject("value-" + i);
+
+ entries.add(new DataEntry(cctx.cacheId(), key, val, op, null, cctx.versions().next(), 0L,
+ cctx.affinity().partition(i), i));
+ }
+
+ UUID cpId = UUID.randomUUID();
+
+ WALPointer start = wal.log(new CheckpointRecord(cpId, null, false));
+
+ wal.fsync(start);
+
+ for (DataEntry entry : entries)
+ wal.log(new DataRecord(entry));
+
+ WALPointer end = wal.log(new CheckpointRecord(cpId, start, true));
+
+ wal.fsync(end);
+
+ // Data will not be written to the page store.
+ stopAllGrids();
+
+ ig = startGrid(0);
+
+ sharedCtx = ig.context().cache().context();
+ cctx = sharedCtx.cache().cache(cacheName).context();
+
+ db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+ wal = sharedCtx.wal();
+
+ db.enableCheckpoints(false);
+
+ try (WALIterator it = wal.replay(start)) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+ assert tup.get2() instanceof CheckpointRecord;
+
+ assertEquals(start, tup.get1());
+
+ CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
+
+ assertEquals(cpId, cpRec.checkpointId());
+ assertNull(cpRec.checkpointMark());
+ assertFalse(cpRec.end());
+
+ int idx = 0;
+ CacheObjectContext coctx = cctx.cacheObjectContext();
+
+ while (idx < entries.size()) {
+ tup = it.nextX();
+
+ assert tup.get2() instanceof DataRecord;
+
+ DataRecord dataRec = (DataRecord)tup.get2();
+
+ DataEntry entry = entries.get(idx);
+
+ assertEquals(1, dataRec.writeEntries().size());
+
+ DataEntry readEntry = dataRec.writeEntries().get(0);
+
+ assertEquals(entry.cacheId(), readEntry.cacheId());
+ assertEquals(entry.key().<Integer>value(coctx, true), readEntry.key().<Integer>value(coctx, true));
+ assertEquals(entry.op(), readEntry.op());
+
+ if (entry.op() == GridCacheOperation.UPDATE)
+ assertEquals(entry.value().value(coctx, true), readEntry.value().value(coctx, true));
+ else
+ assertNull(entry.value());
+
+ assertEquals(entry.writeVersion(), readEntry.writeVersion());
+ assertEquals(entry.nearXidVersion(), readEntry.nearXidVersion());
+ assertEquals(entry.partitionCounter(), readEntry.partitionCounter());
+
+ idx++;
+ }
+
+ tup = it.nextX();
+
+ assert tup.get2() instanceof CheckpointRecord;
+
+ assertEquals(end, tup.get1());
+
+ cpRec = (CheckpointRecord)tup.get2();
+
+ assertEquals(cpId, cpRec.checkpointId());
+ assertEquals(start, cpRec.checkpointMark());
+ assertTrue(cpRec.end());
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testPageWalEntries() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ig.context().cache().context();
+ int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+ PageMemory pageMem = sharedCtx.database().memoryPolicy(null).pageMemory();
+ IgniteWriteAheadLogManager wal = sharedCtx.wal();
+
+ db.enableCheckpoints(false).get();
+
+ int pageCnt = 100;
+
+ List<FullPageId> pageIds = new ArrayList<>();
+
+ for (int i = 0; i < pageCnt; i++) {
+ db.checkpointReadLock();
+ try {
+ pageIds.add(new FullPageId(pageMem.allocatePage(cacheId, PageIdAllocator.INDEX_PARTITION,
+ PageIdAllocator.FLAG_IDX), cacheId));
+ }
+ finally {
+ db.checkpointReadUnlock();
+ }
+ }
+
+ UUID cpId = UUID.randomUUID();
+
+ WALPointer start = wal.log(new CheckpointRecord(cpId, null, false));
+
+ wal.fsync(start);
+
+ ig.context().cache().context().database().checkpointReadLock();
+
+ try {
+ for (FullPageId pageId : pageIds)
+ writePageData(pageId, pageMem);
+ }
+ finally {
+ ig.context().cache().context().database().checkpointReadUnlock();
+ }
+
+ WALPointer end = wal.log(new CheckpointRecord(cpId, start, true));
+
+ wal.fsync(end);
+
+ // Data will not be written to the page store.
+ stopAllGrids();
+
+ ig = startGrid(0);
+
+ sharedCtx = ig.context().cache().context();
+
+ db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+ wal = sharedCtx.wal();
+
+ db.enableCheckpoints(false);
+
+ try (WALIterator it = wal.replay(start)) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+ assert tup.get2() instanceof CheckpointRecord : tup.get2();
+
+ assertEquals(start, tup.get1());
+
+ CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
+
+ assertEquals(cpId, cpRec.checkpointId());
+ assertNull(cpRec.checkpointMark());
+ assertFalse(cpRec.end());
+
+ int idx = 0;
+
+ while (idx < pageIds.size()) {
+ tup = it.nextX();
+
+ assert tup.get2() instanceof PageSnapshot : tup.get2().getClass();
+
+ PageSnapshot snap = (PageSnapshot)tup.get2();
+
+ //there are extra tracking pages, skip them
+ if (TrackingPageIO.VERSIONS.latest().trackingPageFor(snap.fullPageId().pageId(), pageMem.pageSize()) == snap.fullPageId().pageId()) {
+ tup = it.nextX();
+
+ assertTrue(tup.get2() instanceof PageSnapshot);
+
+ snap = (PageSnapshot)tup.get2();
+ }
+
+ assertEquals(pageIds.get(idx), snap.fullPageId());
+
+ idx++;
+ }
+
+ tup = it.nextX();
+
+ assert tup.get2() instanceof CheckpointRecord;
+
+ assertEquals(end, tup.get1());
+
+ cpRec = (CheckpointRecord)tup.get2();
+
+ assertEquals(cpId, cpRec.checkpointId());
+ assertEquals(start, cpRec.checkpointMark());
+ assertTrue(cpRec.end());
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testDirtyFlag() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
+
+ int cacheId = shared.cache().cache(cacheName).context().cacheId();
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
+
+ // Disable integrated checkpoint thread.
+ dbMgr.enableCheckpoints(false);
+
+ PageMemoryEx mem = (PageMemoryEx) dbMgr.memoryPolicy(null).pageMemory();
+
+ ig.context().cache().context().database().checkpointReadLock();
+
+ FullPageId[] pageIds = new FullPageId[100];
+
+ try {
+ for (int i = 0; i < pageIds.length; i++)
+ pageIds[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+
+ for (FullPageId fullId : pageIds) {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page)); //page is dirty right after allocation
+
+ long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ PageIO.setPageId(pageAddr, fullId.pageId());
+
+ try {
+ assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+ }
+ finally {
+ mem.writeUnlock(fullId.cacheId(), fullId.pageId(),page, null,true);
+ }
+
+ assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ }
+ finally {
+ ig.context().cache().context().database().checkpointReadUnlock();
+ }
+
+ Collection<FullPageId> cpPages = mem.beginCheckpoint();
+
+ ig.context().cache().context().database().checkpointReadLock();
+
+ try {
+ for (FullPageId fullId : pageIds) {
+ assertTrue(cpPages.contains(fullId));
+
+ ByteBuffer buf = ByteBuffer.allocate(mem.pageSize());
+
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+
+ long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ try {
+ assertFalse(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
+ PageUtils.putByte(pageAddr, i, (byte)1);
+ }
+ finally {
+ mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
+ }
+
+ assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+
+ buf.rewind();
+
+ mem.getForCheckpoint(fullId, buf, null);
+
+ buf.position(PageIO.COMMON_HEADER_END);
+
+ while (buf.hasRemaining())
+ assertEquals((byte)0, buf.get());
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ }
+ finally {
+ ig.context().cache().context().database().checkpointReadUnlock();
+ }
+
+ mem.finishCheckpoint();
+
+ for (FullPageId fullId : pageIds) {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+ try {
+ assertTrue(mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ private void writePageData(FullPageId fullId, PageMemory mem) throws Exception {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+ try {
+ long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ try {
+ DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
+ PageUtils.putByte(pageAddr, i, (byte)rnd.nextInt(255));
+ }
+ finally {
+ mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+
+ /**
+ * @param res Result map to verify.
+ * @param mem Memory.
+ */
+ private void verifyReads(
+ Map<FullPageId, Integer> res,
+ PageMemory mem,
+ WALPointer start,
+ IgniteWriteAheadLogManager wal
+ ) throws Exception {
+ Map<FullPageId, byte[]> replay = new HashMap<>();
+
+ try (WALIterator it = wal.replay(start)) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
+
+ assertTrue("Invalid record: " + tup, tup.get2() instanceof CheckpointRecord);
+
+ CheckpointRecord cpRec = (CheckpointRecord)tup.get2();
+
+ while (it.hasNextX()) {
+ tup = it.nextX();
+
+ WALRecord rec = tup.get2();
+
+ if (rec instanceof CheckpointRecord) {
+ CheckpointRecord end = (CheckpointRecord)rec;
+
+ // Found the finish mark.
+ if (end.checkpointId().equals(cpRec.checkpointId()) && end.end())
+ break;
+ }
+ else if (rec instanceof PageSnapshot) {
+ PageSnapshot page = (PageSnapshot)rec;
+
+ replay.put(page.fullPageId(), page.pageData());
+ }
+ }
+ }
+
+ // Check read-through from the file store.
+ for (Map.Entry<FullPageId, Integer> entry : res.entrySet()) {
+ FullPageId fullId = entry.getKey();
+ int state = entry.getValue();
+
+ if (state == -1) {
+ info("Page was never written: " + fullId);
+
+ continue;
+ }
+
+ byte[] walData = replay.get(fullId);
+
+ assertNotNull("Missing WAL record for a written page: " + fullId, walData);
+
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+ try {
+ long pageAddr = mem.readLock(fullId.cacheId(), fullId.pageId(), page);
+
+ try {
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
+ assertEquals("Invalid state [pageId=" + fullId + ", pos=" + i + ']',
+ state & 0xFF, PageUtils.getByte(pageAddr, i) & 0xFF);
+
+ assertEquals("Invalid WAL state [pageId=" + fullId + ", pos=" + i + ']',
+ state & 0xFF, walData[i] & 0xFF);
+ }
+ }
+ finally {
+ mem.readUnlock(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ }
+
+ /**
+ * @param mem Memory to use.
+ * @param storeMgr Store manager.
+ * @param cacheId Cache ID.
+ * @return Result map of random operations.
+ * @throws Exception If failure occurred.
+ */
+ private IgniteBiTuple<Map<FullPageId, Integer>, WALPointer> runCheckpointing(
+ final IgniteEx ig,
+ final PageMemoryImpl mem,
+ final IgnitePageStoreManager storeMgr,
+ final IgniteWriteAheadLogManager wal,
+ final int cacheId
+ ) throws Exception {
+ final ConcurrentMap<FullPageId, Integer> resMap = new ConcurrentHashMap<>();
+
+ final FullPageId pages[] = new FullPageId[TOTAL_PAGES];
+
+ Set<FullPageId> allocated = new HashSet<>();
+
+ for (int i = 0; i < TOTAL_PAGES; i++) {
+ FullPageId fullId = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
+
+ resMap.put(fullId, -1);
+
+ pages[i] = fullId;
+
+ allocated.add(fullId);
+ }
+
+ final AtomicBoolean run = new AtomicBoolean(true);
+
+ // Simulate transaction lock.
+ final ReadWriteLock updLock = new ReentrantReadWriteLock();
+
+ // Mark the start position.
+ CheckpointRecord cpRec = new CheckpointRecord(null, false);
+
+ WALPointer start = wal.log(cpRec);
+
+ wal.fsync(start);
+
+ IgniteInternalFuture<Long> updFut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ while (true) {
+ FullPageId fullId = pages[ThreadLocalRandom.current().nextInt(TOTAL_PAGES)];
+
+ updLock.readLock().lock();
+
+ try {
+ if (!run.get())
+ return null;
+
+ ig.context().cache().context().database().checkpointReadLock();
+
+ try {
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
+
+ PageIO.setPageId(pageAddr, fullId.pageId());
+
+ try {
+ int state = resMap.get(fullId);
+
+ if (state != -1) {
+ if (VERBOSE)
+ info("Verify page [fullId=" + fullId + ", state=" + state +
+ ", buf=" + pageAddr +
+ ", bhc=" + U.hexLong(System.identityHashCode(pageAddr)) +
+ ", page=" + U.hexLong(System.identityHashCode(page)) + ']');
+
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
+ assertEquals("Verify page failed [fullId=" + fullId +
+ ", i=" + i +
+ ", state=" + state +
+ ", buf=" + pageAddr +
+ ", bhc=" + U.hexLong(System.identityHashCode(pageAddr)) +
+ ", page=" + U.hexLong(System.identityHashCode(page)) + ']',
+ state & 0xFF, PageUtils.getByte(pageAddr, i) & 0xFF);
+ }
+
+ state = (state + 1) & 0xFF;
+
+ if (VERBOSE)
+ info("Write page [fullId=" + fullId + ", state=" + state +
+ ", buf=" + pageAddr +
+ ", bhc=" + U.hexLong(System.identityHashCode(pageAddr)) +
+ ", page=" + U.hexLong(System.identityHashCode(page)) + ']');
+
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
+ PageUtils.putByte(pageAddr, i, (byte)state);
+
+ resMap.put(fullId, state);
+ }
+ finally {
+ mem.writeUnlock(fullId.cacheId(), fullId.pageId(),page, null,true);
+ }
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(),page);}
+ }
+ finally {
+ ig.context().cache().context().database().checkpointReadUnlock();
+ }
+ }
+ finally {
+ updLock.readLock().unlock();
+ }
+ }
+ }
+ }, 8, "update-thread");
+
+ int checkpoints = 20;
+
+ while (checkpoints > 0) {
+ Map<FullPageId, Integer> snapshot = null;
+
+ Collection<FullPageId> pageIds;
+
+ updLock.writeLock().lock();
+
+ try {
+ snapshot = new HashMap<>(resMap);
+
+ pageIds = mem.beginCheckpoint();
+
+ checkpoints--;
+
+ if (checkpoints == 0)
+ // No more writes should be done at this point.
+ run.set(false);
+
+ info("Acquired pages for checkpoint: " + pageIds.size());
+ }
+ finally {
+ updLock.writeLock().unlock();
+ }
+
+ boolean ok = false;
+
+ try {
+ ByteBuffer tmpBuf = ByteBuffer.allocate(mem.pageSize());
+
+ tmpBuf.order(ByteOrder.nativeOrder());
+
+ long begin = System.currentTimeMillis();
+
+ long cp = 0;
+
+ long write = 0;
+
+ for (FullPageId fullId : pageIds) {
+ long cpStart = System.nanoTime();
+
+ Integer tag = mem.getForCheckpoint(fullId, tmpBuf, null);
+
+ if (tag == null)
+ continue;
+
+ long cpEnd = System.nanoTime();
+
+ cp += cpEnd - cpStart;
+
+ Integer state = snapshot.get(fullId);
+
+ if (allocated.contains(fullId) && state != -1) {
+ tmpBuf.rewind();
+
+ Integer first = null;
+
+ for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++) {
+ int val = tmpBuf.get(i) & 0xFF;
+
+ if (first == null)
+ first = val;
+
+ // Avoid string concat.
+ if (first != val)
+ assertEquals("Corrupted buffer at position [pageId=" + fullId + ", pos=" + i + ']',
+ (int)first, val);
+
+ // Avoid string concat.
+ if (state != val)
+ assertEquals("Invalid value at position [pageId=" + fullId + ", pos=" + i + ']',
+ (int)state, val);
+ }
+ }
+
+ tmpBuf.rewind();
+
+ long writeStart = System.nanoTime();
+
+ storeMgr.write(cacheId, fullId.pageId(), tmpBuf, tag);
+
+ long writeEnd = System.nanoTime();
+
+ write += writeEnd - writeStart;
+
+ tmpBuf.rewind();
+ }
+
+ long syncStart = System.currentTimeMillis();
+
+ storeMgr.sync(cacheId, 0);
+
+ long end = System.currentTimeMillis();
+
+ info("Written pages in " + (end - begin) + "ms, copy took " + (cp / 1_000_000) + "ms, " +
+ "write took " + (write / 1_000_000) + "ms, sync took " + (end - syncStart) + "ms");
+
+ ok = true;
+ }
+ finally {
+ info("Finishing checkpoint...");
+
+ mem.finishCheckpoint();
+
+ info("Finished checkpoint");
+
+ if (!ok) {
+ info("Cancelling updates...");
+
+ run.set(false);
+
+ updFut.get();
+ }
+ }
+
+ if (checkpoints != 0)
+ Thread.sleep(2_000);
+ }
+
+ info("checkpoints=" + checkpoints + ", done=" + updFut.isDone());
+
+ updFut.get();
+
+ // Mark the end.
+ wal.fsync(wal.log(new CheckpointRecord(cpRec.checkpointId(), start, true)));
+
+ assertEquals(0, mem.activePagesCount());
+
+ for (FullPageId fullId : pages) {
+
+ long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
+
+ try {
+ assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" +
+ fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page));
+
+ assertFalse("Page is dirty after the last checkpoint: [cacheId=" +
+ fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
+ }
+ finally {
+ mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+
+ return F.t((Map<FullPageId, Integer>)resMap, start);
+ }
+
+ /**
+ *
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
new file mode 100644
index 0000000..6ca0a1d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgnitePdsWalTlbTest.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import javax.cache.CacheException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.configuration.PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE;
+
+/**
+ *
+ */
+public class IgnitePdsWalTlbTest extends GridCommonAbstractTest {
+ /** Ip finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, Integer> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(100 * 1024 * 1024);
+ memPlcCfg.setMaxSize(100 * 1024 * 1024);
+
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setCheckpointingPageBufferSize(DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE + 1)
+ .setTlbSize(640000000)
+ );
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(IP_FINDER);
+
+ if (gridName.endsWith("1"))
+ cfg.setClientMode(true);
+
+ cfg.setDiscoverySpi(discoSpi);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 30_000;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ stopAllGrids();
+
+ startGrids(2);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalDirectOutOfMemory() throws Exception {
+ IgniteEx ig = grid(1);
+
+ boolean locked = true;
+
+ try {
+ IgniteDataStreamer<Integer, Integer> streamer = ig.dataStreamer(CACHE_NAME);
+
+ for (int i = 0; i < 100_000; i++) {
+ streamer.addData(i, 1);
+
+ if (i > 0 && i % 10_000 == 0)
+ info("Done put: " + i);
+ }
+ }
+ catch (CacheException ignore) {
+ // expected
+ locked = false;
+ }
+ finally {
+ assertFalse(locked);
+
+ stopAllGrids();
+ }
+ }
+}
[15/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheOffheapManager.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheOffheapManager.java
deleted file mode 100644
index c0176f6..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheOffheapManager.java
+++ /dev/null
@@ -1,1316 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageInitRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSnapshotId;
-import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.CacheObject;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl;
-import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeListImpl;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageMetaIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PagePartitionCountersIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PagePartitionMetaIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseListImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionMap;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.lang.GridCursor;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * Used when persistence enabled.
- */
-public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl implements DbCheckpointListener {
- /** */
- private MetaStore metaStore;
-
- /** */
- private ReuseListImpl reuseList;
-
- /** {@inheritDoc} */
- @Override protected void initDataStructures() throws IgniteCheckedException {
- Metas metas = getOrAllocateCacheMetas();
-
- RootPage reuseListRoot = metas.reuseListRoot;
-
- reuseList = new ReuseListImpl(grp.groupId(),
- grp.cacheOrGroupName(),
- grp.memoryPolicy().pageMemory(),
- ctx.wal(),
- reuseListRoot.pageId().pageId(),
- reuseListRoot.isAllocated());
-
- RootPage metastoreRoot = metas.treeRoot;
-
- metaStore = new MetadataStorage(grp.memoryPolicy().pageMemory(),
- ctx.wal(),
- globalRemoveId(),
- grp.groupId(),
- PageIdAllocator.INDEX_PARTITION,
- PageIdAllocator.FLAG_IDX,
- reuseList,
- metastoreRoot.pageId().pageId(),
- metastoreRoot.isAllocated());
-
- ((GridCacheDatabaseSharedManager)ctx.database()).addCheckpointListener(this);
- }
-
- /** {@inheritDoc} */
- @Override public void onCacheStarted(GridCacheContext cctx) throws IgniteCheckedException {
- if (cctx.affinityNode() && cctx.ttl().eagerTtlEnabled() && pendingEntries == null) {
- ctx.database().checkpointReadLock();
-
- try {
- final String name = "PendingEntries";
-
- RootPage pendingRootPage = metaStore.getOrAllocateForTree(name);
-
- pendingEntries = new PendingEntriesTree(
- grp,
- name,
- grp.memoryPolicy().pageMemory(),
- pendingRootPage.pageId().pageId(),
- reuseList,
- pendingRootPage.isAllocated()
- );
- }
- finally {
- ctx.database().checkpointReadUnlock();
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override protected CacheDataStore createCacheDataStore0(final int p)
- throws IgniteCheckedException {
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)ctx.database();
-
- boolean exists = ctx.pageStore() != null
- && ctx.pageStore().exists(grp.groupId(), p);
-
- return new GridCacheDataStore(p, exists);
- }
-
- /** {@inheritDoc} */
- @Override public void onCheckpointBegin(Context ctx) throws IgniteCheckedException {
- assert grp.memoryPolicy().pageMemory() instanceof PageMemoryEx;
-
- reuseList.saveMetadata();
-
- boolean metaWasUpdated = false;
-
- for (CacheDataStore store : partDataStores.values()) {
- RowStore rowStore = store.rowStore();
-
- if (rowStore == null)
- continue;
-
- metaWasUpdated |= saveStoreMetadata(store, ctx, !metaWasUpdated, false);
- }
- }
-
- /**
- * @param store Store to save metadata.
- * @throws IgniteCheckedException If failed.
- */
- private boolean saveStoreMetadata(CacheDataStore store, Context ctx, boolean saveMeta,
- boolean beforeDestroy) throws IgniteCheckedException {
- RowStore rowStore0 = store.rowStore();
-
- boolean beforeSnapshot = ctx != null && ctx.nextSnapshot();
-
- boolean wasSaveToMeta = false;
-
- if (rowStore0 != null) {
- FreeListImpl freeList = (FreeListImpl)rowStore0.freeList();
-
- freeList.saveMetadata();
-
- long updCntr = store.updateCounter();
- int size = store.fullSize();
- long rmvId = globalRemoveId().get();
-
- PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
- IgniteWriteAheadLogManager wal = this.ctx.wal();
-
- if (size > 0 || updCntr > 0) {
- int state = -1;
-
- if (beforeDestroy)
- state = GridDhtPartitionState.EVICTED.ordinal();
- else {
- // localPartition will not acquire writeLock here because create=false.
- GridDhtLocalPartition part = grp.topology().localPartition(store.partId(),
- AffinityTopologyVersion.NONE, false);
-
- if (part != null && part.state() != GridDhtPartitionState.EVICTED)
- state = part.state().ordinal();
- }
-
- // Do not save meta for evicted partitions on next checkpoints.
- if (state == -1)
- return false;
-
- int grpId = grp.groupId();
- long partMetaId = pageMem.partitionMetaPageId(grpId, store.partId());
- long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
-
- try {
- long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
-
- if (pageAddr == 0L) {
- U.warn(log, "Failed to acquire write lock for meta page [metaPage=" + partMetaPage +
- ", saveMeta=" + saveMeta + ", beforeDestroy=" + beforeDestroy + ", size=" + size +
- ", updCntr=" + updCntr + ", state=" + state + ']');
-
- return false;
- }
-
- boolean changed = false;
-
- try {
- PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
-
- changed |= io.setUpdateCounter(pageAddr, updCntr);
- changed |= io.setGlobalRemoveId(pageAddr, rmvId);
- changed |= io.setSize(pageAddr, size);
-
- changed |= io.setPartitionState(pageAddr, (byte)state);
-
- long cntrsPageId;
-
- if (grp.sharedGroup()) {
- cntrsPageId = io.getCountersPageId(pageAddr);
-
- byte[] data = serializeCacheSizes(store.cacheSizes());
-
- int items = data.length / 12;
- int written = 0;
- int pageSize = pageMem.pageSize();
-
- boolean init = cntrsPageId == 0;
-
- if (init) {
- cntrsPageId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
-
- io.setCountersPageId(pageAddr, cntrsPageId);
-
- changed = true;
- }
-
- long nextId = cntrsPageId;
-
- while (written != items) {
- final long curId = nextId;
- final long curPage = pageMem.acquirePage(grpId, curId);
-
- try {
- final long curAddr = pageMem.writeLock(grpId, curId, curPage);
-
- assert curAddr != 0;
-
- try {
- PagePartitionCountersIO partMetaIo;
-
- if (init) {
- partMetaIo = PagePartitionCountersIO.VERSIONS.latest();
-
- partMetaIo.initNewPage(curAddr, curId, pageSize);
- }
- else
- partMetaIo = PageIO.getPageIO(curAddr);
-
- written += partMetaIo.writeCacheSizes(pageSize, curAddr, data, written);
-
- nextId = partMetaIo.getNextCountersPageId(curAddr);
-
- if(written != items && (init = nextId == 0)) {
- //allocate new counters page
- nextId = pageMem.allocatePage(grpId, store.partId(), PageIdAllocator.FLAG_DATA);
- partMetaIo.setNextCountersPageId(curAddr, nextId);
- }
- }
- finally {
- // Write full page
- pageMem.writeUnlock(grpId, curId, curPage, Boolean.TRUE, true);
- }
- }
- finally {
- pageMem.releasePage(grpId, curId, curPage);
- }
- }
- }
- else
- cntrsPageId = 0L;
-
- int pageCnt;
-
- if (beforeSnapshot) {
- pageCnt = this.ctx.pageStore().pages(grpId, store.partId());
- io.setCandidatePageCount(pageAddr, pageCnt);
-
- if (saveMeta) {
- long metaPageId = pageMem.metaPageId(grpId);
- long metaPage = pageMem.acquirePage(grpId, metaPageId);
-
- try {
- long metaPageAddr = pageMem.writeLock(grpId, metaPageId, metaPage);
-
- try {
- long nextSnapshotTag = io.getNextSnapshotTag(metaPageAddr);
-
- io.setNextSnapshotTag(metaPageAddr, nextSnapshotTag + 1);
-
- if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaPageId,
- metaPage, wal, null))
- wal.log(new MetaPageUpdateNextSnapshotId(grpId, metaPageId,
- nextSnapshotTag + 1));
-
- addPartition(ctx.partitionStatMap(), metaPageAddr, io, grpId, PageIdAllocator.INDEX_PARTITION,
- this.ctx.kernalContext().cache().context().pageStore().pages(grpId, PageIdAllocator.INDEX_PARTITION));
- }
- finally {
- pageMem.writeUnlock(grpId, metaPageId, metaPage, null, true);
- }
- }
- finally {
- pageMem.releasePage(grpId, metaPageId, metaPage);
- }
-
- wasSaveToMeta = true;
- }
-
- GridDhtPartitionMap partMap = grp.topology().localPartitionMap();
-
- if (partMap.containsKey(store.partId()) &&
- partMap.get(store.partId()) == GridDhtPartitionState.OWNING)
- addPartition(ctx.partitionStatMap(), pageAddr, io, grpId, store.partId(),
- this.ctx.pageStore().pages(grpId, store.partId()));
-
- changed = true;
- }
- else
- pageCnt = io.getCandidatePageCount(pageAddr);
-
- if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
- wal.log(new MetaPageUpdatePartitionDataRecord(
- grpId,
- partMetaId,
- updCntr,
- rmvId,
- size,
- cntrsPageId,
- (byte)state,
- pageCnt
- ));
- }
- finally {
- pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
- }
- }
- finally {
- pageMem.releasePage(grpId, partMetaId, partMetaPage);
- }
- }
- }
-
- return wasSaveToMeta;
- }
-
- /**
- * @param cacheSizes Cache sizes.
- * @return Serialized cache sizes
- */
- private byte[] serializeCacheSizes(Map<Integer, Long> cacheSizes) {
- // Item size = 4 bytes (cache ID) + 8 bytes (cache size) = 12 bytes
- byte[] data = new byte[cacheSizes.size() * 12];
- long off = GridUnsafe.BYTE_ARR_OFF;
-
- for (Map.Entry<Integer, Long> entry : cacheSizes.entrySet()) {
- GridUnsafe.putInt(data, off, entry.getKey()); off += 4;
- GridUnsafe.putLong(data, off, entry.getValue()); off += 8;
- }
-
- return data;
- }
-
- /**
- * @param map Map to add values to.
- * @param pageAddr page address
- * @param io Page Meta IO
- * @param cacheId Cache ID.
- * @param partition Partition ID.
- * @param pages Number of pages to add.
- */
- private static void addPartition(
- Map<T2<Integer, Integer>, T2<Integer, Integer>> map,
- long pageAddr,
- PagePartitionMetaIO io,
- int cacheId,
- int partition,
- int pages
- ) {
- if (pages <= 1)
- return;
-
- assert PageIO.getPageId(pageAddr) != 0;
-
- int lastAllocatedIdx = io.getLastPageCount(pageAddr);
- map.put(new T2<>(cacheId, partition), new T2<>(lastAllocatedIdx, pages));
- }
-
- /** {@inheritDoc} */
- @Override protected void destroyCacheDataStore0(CacheDataStore store) throws IgniteCheckedException {
- ctx.database().checkpointReadLock();
-
- try {
- int p = store.partId();
-
- saveStoreMetadata(store, null, false, true);
-
- PageMemoryEx pageMemory = (PageMemoryEx)grp.memoryPolicy().pageMemory();
-
- int tag = pageMemory.invalidate(grp.groupId(), p);
-
- ctx.wal().log(new PartitionDestroyRecord(grp.groupId(), p));
-
- ctx.pageStore().onPartitionDestroyed(grp.groupId(), p, tag);
- }
- finally {
- ctx.database().checkpointReadUnlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void onPartitionCounterUpdated(int part, long cntr) {
- CacheDataStore store = partDataStores.get(part);
-
- assert store != null;
-
- long oldCnt = store.updateCounter();
-
- if (oldCnt < cntr)
- store.updateCounter(cntr);
- }
-
- /** {@inheritDoc} */
- @Override public void onPartitionInitialCounterUpdated(int part, long cntr) {
- CacheDataStore store = partDataStores.get(part);
-
- assert store != null;
-
- long oldCnt = store.initialUpdateCounter();
-
- if (oldCnt < cntr)
- store.updateInitialCounter(cntr);
- }
-
- /** {@inheritDoc} */
- @Override public long lastUpdatedPartitionCounter(int part) {
- return partDataStores.get(part).updateCounter();
- }
-
- /** {@inheritDoc} */
- @Override public RootPage rootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException {
- if (grp.sharedGroup())
- idxName = Integer.toString(cacheId) + "_" + idxName;
-
- return metaStore.getOrAllocateForTree(idxName);
- }
-
- /** {@inheritDoc} */
- @Override public void dropRootPageForIndex(int cacheId, String idxName) throws IgniteCheckedException {
- if (grp.sharedGroup())
- idxName = Integer.toString(cacheId) + "_" + idxName;
-
- metaStore.dropRootPage(idxName);
- }
-
- /** {@inheritDoc} */
- @Override public ReuseList reuseListForIndex(String idxName) {
- return reuseList;
- }
-
- /** {@inheritDoc} */
- @Override public void stop() {
- if (grp.affinityNode())
- ((GridCacheDatabaseSharedManager)ctx.database()).removeCheckpointListener(this);
- }
-
- /**
- * @return Meta root pages info.
- * @throws IgniteCheckedException If failed.
- */
- private Metas getOrAllocateCacheMetas() throws IgniteCheckedException {
- PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
- IgniteWriteAheadLogManager wal = ctx.wal();
-
- int grpId = grp.groupId();
- long metaId = pageMem.metaPageId(grpId);
- long metaPage = pageMem.acquirePage(grpId, metaId);
-
- try {
- final long pageAddr = pageMem.writeLock(grpId, metaId, metaPage);
-
- boolean allocated = false;
-
- try {
- long metastoreRoot, reuseListRoot;
-
- if (PageIO.getType(pageAddr) != PageIO.T_META) {
- PageMetaIO pageIO = PageMetaIO.VERSIONS.latest();
-
- pageIO.initNewPage(pageAddr, metaId, pageMem.pageSize());
-
- metastoreRoot = pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX);
- reuseListRoot = pageMem.allocatePage(grpId, PageIdAllocator.INDEX_PARTITION, PageMemory.FLAG_IDX);
-
- pageIO.setTreeRoot(pageAddr, metastoreRoot);
- pageIO.setReuseListRoot(pageAddr, reuseListRoot);
-
- if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, metaId, metaPage, wal, null))
- wal.log(new MetaPageInitRecord(
- grpId,
- metaId,
- pageIO.getType(),
- pageIO.getVersion(),
- metastoreRoot,
- reuseListRoot
- ));
-
- allocated = true;
- }
- else {
- PageMetaIO pageIO = PageIO.getPageIO(pageAddr);
-
- metastoreRoot = pageIO.getTreeRoot(pageAddr);
- reuseListRoot = pageIO.getReuseListRoot(pageAddr);
-
- assert reuseListRoot != 0L;
- }
-
- return new Metas(
- new RootPage(new FullPageId(metastoreRoot, grpId), allocated),
- new RootPage(new FullPageId(reuseListRoot, grpId), allocated));
- }
- finally {
- pageMem.writeUnlock(grpId, metaId, metaPage, null, allocated);
- }
- }
- finally {
- pageMem.releasePage(grpId, metaId, metaPage);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgniteRebalanceIterator rebalanceIterator(int part, AffinityTopologyVersion topVer,
- Long partCntrSince) throws IgniteCheckedException {
- if (partCntrSince == null)
- return super.rebalanceIterator(part, topVer, partCntrSince);
-
- GridCacheDatabaseSharedManager database = (GridCacheDatabaseSharedManager)ctx.database();
-
- try {
- WALPointer startPtr = database.searchPartitionCounter(grp.groupId(), part, partCntrSince);
-
- if (startPtr == null) {
- assert false : "partCntr=" + partCntrSince + ", reservations=" + S.toString(Map.class, database.reservedForPreloading());
-
- return super.rebalanceIterator(part, topVer, partCntrSince);
- }
-
- WALIterator it = ctx.wal().replay(startPtr);
-
- return new RebalanceIteratorAdapter(grp, it, part);
- }
- catch (IgniteCheckedException e) {
- U.warn(log, "Failed to create WAL-based rebalance iterator (a full partition will transferred to a " +
- "remote node) [part=" + part + ", partCntrSince=" + partCntrSince + ", err=" + e + ']');
-
- return super.rebalanceIterator(part, topVer, partCntrSince);
- }
- }
-
- /**
- *
- */
- private static class RebalanceIteratorAdapter implements IgniteRebalanceIterator {
- /** Serial version uid. */
- private static final long serialVersionUID = 0L;
-
- /** Cache group caches. */
- private final Set<Integer> cacheGrpCaches;
-
- /** WAL iterator. */
- private final WALIterator walIt;
-
- /** Partition to scan. */
- private final int part;
-
- /** */
- private Iterator<DataEntry> entryIt;
-
- /** */
- private CacheDataRow next;
-
- /**
- * @param grp Cache group.
- * @param walIt WAL iterator.
- * @param part Partition ID.
- */
- private RebalanceIteratorAdapter(CacheGroupContext grp, WALIterator walIt, int part) {
- this.cacheGrpCaches = grp.cacheIds();
- this.walIt = walIt;
- this.part = part;
-
- advance();
- }
-
- /** {@inheritDoc} */
- @Override public boolean historical() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- walIt.close();
- }
-
- /** {@inheritDoc} */
- @Override public boolean isClosed() {
- return walIt.isClosed();
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasNextX() {
- return hasNext();
- }
-
- /** {@inheritDoc} */
- @Override public CacheDataRow nextX() throws IgniteCheckedException {
- return next();
- }
-
- /** {@inheritDoc} */
- @Override public void removeX() throws IgniteCheckedException {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public Iterator<CacheDataRow> iterator() {
- return this;
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasNext() {
- return next != null;
- }
-
- /** {@inheritDoc} */
- @Override public CacheDataRow next() {
- if (next == null)
- throw new NoSuchElementException();
-
- CacheDataRow val = next;
-
- advance();
-
- return val;
- }
-
- /** {@inheritDoc} */
- @Override public void remove() {
- throw new UnsupportedOperationException();
- }
-
- /**
- *
- */
- private void advance() {
- next = null;
-
- while (true) {
- if (entryIt != null) {
- while (entryIt.hasNext()) {
- DataEntry entry = entryIt.next();
-
- if (entry.partitionId() == part && cacheGrpCaches.contains(entry.cacheId())) {
-
- next = new DataEntryRow(entry);
-
- return;
- }
- }
- }
-
- entryIt = null;
-
- while (walIt.hasNext()) {
- IgniteBiTuple<WALPointer, WALRecord> rec = walIt.next();
-
- if (rec.get2() instanceof DataRecord) {
- DataRecord data = (DataRecord)rec.get2();
-
- entryIt = data.writeEntries().iterator();
- // Move on to the next valid data entry.
-
- break;
- }
- }
-
- if (entryIt == null)
- return;
- }
- }
- }
-
- /**
- * Data entry row.
- */
- private static class DataEntryRow implements CacheDataRow {
- /** */
- private final DataEntry entry;
-
- /**
- * @param entry Data entry.
- */
- private DataEntryRow(DataEntry entry) {
- this.entry = entry;
- }
-
- /** {@inheritDoc} */
- @Override public KeyCacheObject key() {
- return entry.key();
- }
-
- /** {@inheritDoc} */
- @Override public void key(KeyCacheObject key) {
- throw new IllegalStateException();
- }
-
- /** {@inheritDoc} */
- @Override public CacheObject value() {
- return entry.value();
- }
-
- /** {@inheritDoc} */
- @Override public GridCacheVersion version() {
- return entry.writeVersion();
- }
-
- /** {@inheritDoc} */
- @Override public long expireTime() {
- return entry.expireTime();
- }
-
- /** {@inheritDoc} */
- @Override public int partition() {
- return entry.partitionId();
- }
-
- /** {@inheritDoc} */
- @Override public long link() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override public void link(long link) {
- throw new UnsupportedOperationException();
- }
-
- /** {@inheritDoc} */
- @Override public int hash() {
- return entry.key().hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public int cacheId() {
- return entry.cacheId();
- }
- }
-
- /**
- *
- */
- private static class Metas {
- /** */
- @GridToStringInclude
- private final RootPage reuseListRoot;
-
- /** */
- @GridToStringInclude
- private final RootPage treeRoot;
-
- /**
- * @param treeRoot Metadata storage root.
- * @param reuseListRoot Reuse list root.
- */
- Metas(RootPage treeRoot, RootPage reuseListRoot) {
- this.treeRoot = treeRoot;
- this.reuseListRoot = reuseListRoot;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(Metas.class, this);
- }
- }
-
- /**
- *
- */
- private class GridCacheDataStore implements CacheDataStore {
- /** */
- private final int partId;
-
- /** */
- private String name;
-
- /** */
- private volatile FreeListImpl freeList;
-
- /** */
- private volatile CacheDataStore delegate;
-
- /** */
- private final boolean exists;
-
- /** */
- private final AtomicBoolean init = new AtomicBoolean();
-
- /** */
- private final CountDownLatch latch = new CountDownLatch(1);
-
- /**
- * @param partId Partition.
- * @param exists {@code True} if store for this index exists.
- */
- private GridCacheDataStore(int partId, boolean exists) {
- this.partId = partId;
- this.exists = exists;
-
- name = treeName(partId);
- }
-
- /**
- * @return Store delegate.
- * @throws IgniteCheckedException If failed.
- */
- private CacheDataStore init0(boolean checkExists) throws IgniteCheckedException {
- CacheDataStore delegate0 = delegate;
-
- if (delegate0 != null)
- return delegate0;
-
- if (checkExists) {
- if (!exists)
- return null;
- }
-
- IgniteCacheDatabaseSharedManager dbMgr = ctx.database();
-
- dbMgr.checkpointReadLock();
-
- if (init.compareAndSet(false, true)) {
- try {
- Metas metas = getOrAllocatePartitionMetas();
-
- RootPage reuseRoot = metas.reuseListRoot;
-
- freeList = new FreeListImpl(
- grp.groupId(),
- grp.cacheOrGroupName() + "-" + partId,
- grp.memoryPolicy().memoryMetrics(),
- grp.memoryPolicy(),
- null,
- ctx.wal(),
- reuseRoot.pageId().pageId(),
- reuseRoot.isAllocated()) {
- @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
- return pageMem.allocatePage(cacheId, partId, PageIdAllocator.FLAG_DATA);
- }
- };
-
- CacheDataRowStore rowStore = new CacheDataRowStore(grp, freeList, partId);
-
- RootPage treeRoot = metas.treeRoot;
-
- CacheDataTree dataTree = new CacheDataTree(
- grp,
- name,
- freeList,
- rowStore,
- treeRoot.pageId().pageId(),
- treeRoot.isAllocated()) {
- @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
- return pageMem.allocatePage(cacheId, partId, PageIdAllocator.FLAG_DATA);
- }
- };
-
- PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
-
- delegate0 = new CacheDataStoreImpl(partId, name, rowStore, dataTree);
-
- int grpId = grp.groupId();
- long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
- long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
-
- try {
- long pageAddr = pageMem.readLock(grpId, partMetaId, partMetaPage);
-
- try {
- if (PageIO.getType(pageAddr) != 0) {
- PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.latest();
-
- Map<Integer, Long> cacheSizes = null;
-
- if (grp.sharedGroup()) {
- long cntrsPageId = io.getCountersPageId(pageAddr);
-
- if (cntrsPageId != 0L) {
- cacheSizes = new HashMap<>();
-
- long nextId = cntrsPageId;
-
- while (true){
- final long curId = nextId;
- final long curPage = pageMem.acquirePage(grpId, curId);
-
- try {
- final long curAddr = pageMem.readLock(grpId, curId, curPage);
-
- assert curAddr != 0;
-
- try {
- PagePartitionCountersIO cntrsIO = PageIO.getPageIO(curAddr);
-
- if (cntrsIO.readCacheSizes(curAddr, cacheSizes))
- break;
-
- nextId = cntrsIO.getNextCountersPageId(curAddr);
-
- assert nextId != 0;
- }
- finally {
- pageMem.readUnlock(grpId, curId, curPage);
- }
- }
- finally {
- pageMem.releasePage(grpId, curId, curPage);
- }
- }
- }
- }
-
- delegate0.init(io.getSize(pageAddr), io.getUpdateCounter(pageAddr), cacheSizes);
-
- globalRemoveId().setIfGreater(io.getGlobalRemoveId(pageAddr));
- }
- }
- finally {
- pageMem.readUnlock(grpId, partMetaId, partMetaPage);
- }
- }
- finally {
- pageMem.releasePage(grpId, partMetaId, partMetaPage);
- }
-
- delegate = delegate0;
- }
- finally {
- latch.countDown();
-
- dbMgr.checkpointReadUnlock();
- }
- }
- else {
- dbMgr.checkpointReadUnlock();
-
- U.await(latch);
-
- delegate0 = delegate;
-
- if (delegate0 == null)
- throw new IgniteCheckedException("Cache store initialization failed.");
- }
-
- return delegate0;
- }
-
- /**
- * @return Partition metas.
- */
- private Metas getOrAllocatePartitionMetas() throws IgniteCheckedException {
- PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
- IgniteWriteAheadLogManager wal = ctx.wal();
-
- int grpId = grp.groupId();
- long partMetaId = pageMem.partitionMetaPageId(grpId, partId);
- long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
- try {
- boolean allocated = false;
- long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
-
- try {
- long treeRoot, reuseListRoot;
-
- // Initialize new page.
- if (PageIO.getType(pageAddr) != PageIO.T_PART_META) {
- PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.latest();
-
- io.initNewPage(pageAddr, partMetaId, pageMem.pageSize());
-
- treeRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_DATA);
- reuseListRoot = pageMem.allocatePage(grpId, partId, PageMemory.FLAG_DATA);
-
- assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA;
- assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA;
-
- io.setTreeRoot(pageAddr, treeRoot);
- io.setReuseListRoot(pageAddr, reuseListRoot);
-
- if (PageHandler.isWalDeltaRecordNeeded(pageMem, grpId, partMetaId, partMetaPage, wal, null))
- wal.log(new MetaPageInitRecord(
- grpId,
- partMetaId,
- io.getType(),
- io.getVersion(),
- treeRoot,
- reuseListRoot
- ));
-
- allocated = true;
- }
- else {
- PagePartitionMetaIO io = PageIO.getPageIO(pageAddr);
-
- treeRoot = io.getTreeRoot(pageAddr);
- reuseListRoot = io.getReuseListRoot(pageAddr);
-
- assert PageIdUtils.flag(treeRoot) == PageMemory.FLAG_DATA :
- U.hexLong(treeRoot) + ", part=" + partId + ", grpId=" + grpId;
- assert PageIdUtils.flag(reuseListRoot) == PageMemory.FLAG_DATA :
- U.hexLong(reuseListRoot) + ", part=" + partId + ", grpId=" + grpId;
- }
-
- return new Metas(
- new RootPage(new FullPageId(treeRoot, grpId), allocated),
- new RootPage(new FullPageId(reuseListRoot, grpId), allocated));
- }
- finally {
- pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, allocated);
- }
- }
- finally {
- pageMem.releasePage(grpId, partMetaId, partMetaPage);
- }
- }
-
- /** {@inheritDoc} */
- @Override public int partId() {
- return partId;
- }
-
- /** {@inheritDoc} */
- @Override public String name() {
- return name;
- }
-
- /** {@inheritDoc} */
- @Override public RowStore rowStore() {
- CacheDataStore delegate0 = delegate;
-
- return delegate0 == null ? null : delegate0.rowStore();
- }
-
- /** {@inheritDoc} */
- @Override public int fullSize() {
- try {
- CacheDataStore delegate0 = init0(true);
-
- return delegate0 == null ? 0 : delegate0.fullSize();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public int cacheSize(int cacheId) {
- try {
- CacheDataStore delegate0 = init0(true);
-
- return delegate0 == null ? 0 : delegate0.cacheSize(cacheId);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Map<Integer, Long> cacheSizes() {
- try {
- CacheDataStore delegate0 = init0(true);
-
- return delegate0 == null ? null : delegate0.cacheSizes();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public long updateCounter() {
- try {
- CacheDataStore delegate0 = init0(true);
-
- return delegate0 == null ? 0 : delegate0.updateCounter();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void init(long size, long updCntr, @Nullable Map<Integer, Long> cacheSizes) {
- throw new IllegalStateException("Should be never called.");
- }
-
- /** {@inheritDoc} */
- @Override public void updateCounter(long val) {
- try {
- CacheDataStore delegate0 = init0(false);
-
- if (delegate0 != null)
- delegate0.updateCounter(val);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public long nextUpdateCounter() {
- try {
- CacheDataStore delegate0 = init0(false);
-
- return delegate0 == null ? 0 : delegate0.nextUpdateCounter();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Long initialUpdateCounter() {
- try {
- CacheDataStore delegate0 = init0(true);
-
- return delegate0 == null ? 0 : delegate0.initialUpdateCounter();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void updateInitialCounter(long cntr) {
- try {
- CacheDataStore delegate0 = init0(true);
-
- if (delegate0 != null)
- delegate0.updateInitialCounter(cntr);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void update(
- GridCacheContext cctx,
- KeyCacheObject key,
- CacheObject val,
- GridCacheVersion ver,
- long expireTime,
- @Nullable CacheDataRow oldRow
- ) throws IgniteCheckedException {
- CacheDataStore delegate = init0(false);
-
- delegate.update(cctx, key, val, ver, expireTime, oldRow);
- }
-
- /** {@inheritDoc} */
- @Override public void updateIndexes(GridCacheContext cctx, KeyCacheObject key) throws IgniteCheckedException {
- CacheDataStore delegate = init0(false);
-
- delegate.updateIndexes(cctx, key);
- }
-
- /** {@inheritDoc} */
- @Override public CacheDataRow createRow(
- GridCacheContext cctx,
- KeyCacheObject key,
- CacheObject val,
- GridCacheVersion ver,
- long expireTime,
- @Nullable CacheDataRow oldRow) throws IgniteCheckedException {
- CacheDataStore delegate = init0(false);
-
- return delegate.createRow(cctx, key, val, ver, expireTime, oldRow);
- }
-
- /** {@inheritDoc} */
- @Override public void invoke(GridCacheContext cctx, KeyCacheObject key, OffheapInvokeClosure c)
- throws IgniteCheckedException {
- CacheDataStore delegate = init0(false);
-
- delegate.invoke(cctx, key, c);
- }
-
- /** {@inheritDoc} */
- @Override public void remove(GridCacheContext cctx, KeyCacheObject key, int partId)
- throws IgniteCheckedException {
- CacheDataStore delegate = init0(false);
-
- delegate.remove(cctx, key, partId);
- }
-
- /** {@inheritDoc} */
- @Override public CacheDataRow find(GridCacheContext cctx, KeyCacheObject key) throws IgniteCheckedException {
- CacheDataStore delegate = init0(true);
-
- if (delegate != null)
- return delegate.find(cctx, key);
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public GridCursor<? extends CacheDataRow> cursor() throws IgniteCheckedException {
- CacheDataStore delegate = init0(true);
-
- if (delegate != null)
- return delegate.cursor();
-
- return EMPTY_CURSOR;
- }
-
- /** {@inheritDoc} */
- @Override public GridCursor<? extends CacheDataRow> cursor(
- int cacheId,
- KeyCacheObject lower,
- KeyCacheObject upper) throws IgniteCheckedException {
- CacheDataStore delegate = init0(true);
-
- if (delegate != null)
- return delegate.cursor(cacheId, lower, upper);
-
- return EMPTY_CURSOR;
- }
-
- /** {@inheritDoc} */
- @Override public GridCursor<? extends CacheDataRow> cursor(int cacheId,
- KeyCacheObject lower,
- KeyCacheObject upper,
- Object x)
- throws IgniteCheckedException {
- CacheDataStore delegate = init0(true);
-
- if (delegate != null)
- return delegate.cursor(cacheId, lower, upper, x);
-
- return EMPTY_CURSOR;
- }
-
- /** {@inheritDoc} */
- @Override public void destroy() throws IgniteCheckedException {
- // No need to destroy delegate.
- }
-
- /** {@inheritDoc} */
- @Override public GridCursor<? extends CacheDataRow> cursor(int cacheId) throws IgniteCheckedException {
- CacheDataStore delegate = init0(true);
-
- if (delegate != null)
- return delegate.cursor(cacheId);
-
- return EMPTY_CURSOR;
- }
-
- /** {@inheritDoc} */
- @Override public void clear(int cacheId) throws IgniteCheckedException {
- CacheDataStore delegate = init0(true);
-
- if (delegate != null)
- delegate.clear(cacheId);
- }
- }
-
- /**
- *
- */
- private static final GridCursor<CacheDataRow> EMPTY_CURSOR = new GridCursor<CacheDataRow>() {
- /** {@inheritDoc} */
- @Override public boolean next() {
- return false;
- }
-
- /** {@inheritDoc} */
- @Override public CacheDataRow get() {
- return null;
- }
- };
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsImpl.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsImpl.java
deleted file mode 100644
index 25d95ae..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsImpl.java
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
-import org.apache.ignite.mxbean.PersistenceMetricsMXBean;
-
-/**
- *
- */
-public class PersistenceMetricsImpl implements PersistenceMetricsMXBean {
- /** */
- private volatile HitRateMetrics walLoggingRate;
-
- /** */
- private volatile HitRateMetrics walWritingRate;
-
- /** */
- private volatile HitRateMetrics walFsyncTimeDuration;
-
- /** */
- private volatile HitRateMetrics walFsyncTimeNumber;
-
- /** */
- private volatile long lastCpLockWaitDuration;
-
- /** */
- private volatile long lastCpMarkDuration;
-
- /** */
- private volatile long lastCpPagesWriteDuration;
-
- /** */
- private volatile long lastCpDuration;
-
- /** */
- private volatile long lastCpFsyncDuration;
-
- /** */
- private volatile long lastCpTotalPages;
-
- /** */
- private volatile long lastCpDataPages;
-
- /** */
- private volatile long lastCpCowPages;
-
- /** */
- private volatile long rateTimeInterval;
-
- /** */
- private volatile int subInts;
-
- /** */
- private volatile boolean metricsEnabled;
-
- /** */
- private IgniteWriteAheadLogManager wal;
-
- /**
- * @param metricsEnabled Metrics enabled flag.
- * @param rateTimeInterval Rate time interval.
- * @param subInts Number of sub-intervals.
- */
- public PersistenceMetricsImpl(
- boolean metricsEnabled,
- long rateTimeInterval,
- int subInts
- ) {
- this.metricsEnabled = metricsEnabled;
- this.rateTimeInterval = rateTimeInterval;
- this.subInts = subInts;
-
- resetRates();
- }
-
- /** {@inheritDoc} */
- @Override public float getWalLoggingRate() {
- if (!metricsEnabled)
- return 0;
-
- return ((float)walLoggingRate.getRate()) / rateTimeInterval;
- }
-
- /** {@inheritDoc} */
- @Override public float getWalWritingRate() {
- if (!metricsEnabled)
- return 0;
-
- return ((float)walWritingRate.getRate()) / rateTimeInterval;
- }
-
- /** {@inheritDoc} */
- @Override public int getWalArchiveSegments() {
- if (!metricsEnabled)
- return 0;
-
- return wal.walArchiveSegments();
- }
-
- /** {@inheritDoc} */
- @Override public float getWalFsyncTimeAverage() {
- if (!metricsEnabled)
- return 0;
-
- long numRate = walFsyncTimeNumber.getRate();
-
- if (numRate == 0)
- return 0;
-
- return (float)walFsyncTimeDuration.getRate() / numRate;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointingDuration() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointLockWaitDuration() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpLockWaitDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointMarkDuration() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpMarkDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointPagesWriteDuration() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpPagesWriteDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointFsyncDuration() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpFsyncDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointTotalPagesNumber() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpTotalPages;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointDataPagesNumber() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpDataPages;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointCopiedOnWritePagesNumber() {
- if (!metricsEnabled)
- return 0;
-
- return lastCpCowPages;
- }
-
- /** {@inheritDoc} */
- @Override public void enableMetrics() {
- metricsEnabled = true;
- }
-
- /** {@inheritDoc} */
- @Override public void disableMetrics() {
- metricsEnabled = false;
- }
-
- /** {@inheritDoc} */
- @Override public void rateTimeInterval(long rateTimeInterval) {
- this.rateTimeInterval = rateTimeInterval;
-
- resetRates();
- }
-
- /** {@inheritDoc} */
- @Override public void subIntervals(int subInts) {
- this.subInts = subInts;
-
- resetRates();
- }
-
- /**
- * @param wal Write-ahead log manager.
- */
- public void wal(IgniteWriteAheadLogManager wal) {
- this.wal = wal;
- }
-
- /**
- * @return Metrics enabled flag.
- */
- public boolean metricsEnabled() {
- return metricsEnabled;
- }
-
- /**
- * @param lockWaitDuration Lock wait duration.
- * @param markDuration Mark duration.
- * @param pagesWriteDuration Pages write duration.
- * @param fsyncDuration Total checkpoint fsync duration.
- * @param duration Total checkpoint duration.
- * @param totalPages Total number of all pages in checkpoint.
- * @param dataPages Total number of data pages in checkpoint.
- * @param cowPages Total number of COW-ed pages in checkpoint.
- */
- public void onCheckpoint(
- long lockWaitDuration,
- long markDuration,
- long pagesWriteDuration,
- long fsyncDuration,
- long duration,
- long totalPages,
- long dataPages,
- long cowPages
- ) {
- if (metricsEnabled) {
- lastCpLockWaitDuration = lockWaitDuration;
- lastCpMarkDuration = markDuration;
- lastCpPagesWriteDuration = pagesWriteDuration;
- lastCpFsyncDuration = fsyncDuration;
- lastCpDuration = duration;
- lastCpTotalPages = totalPages;
- lastCpDataPages = dataPages;
- lastCpCowPages = cowPages;
- }
- }
-
- /**
- *
- */
- public void onWalRecordLogged() {
- walLoggingRate.onHit();
- }
-
- /**
- * @param size Size written.
- */
- public void onWalBytesWritten(int size) {
- walWritingRate.onHits(size);
- }
-
- /**
- * @param nanoTime Fsync nano time.
- */
- public void onFsync(long nanoTime) {
- long microseconds = nanoTime / 1_000;
-
- walFsyncTimeDuration.onHits(microseconds);
- walFsyncTimeNumber.onHit();
- }
-
- /**
- *
- */
- private void resetRates() {
- walLoggingRate = new HitRateMetrics((int)rateTimeInterval, subInts);
- walWritingRate = new HitRateMetrics((int)rateTimeInterval, subInts);
-
- walFsyncTimeDuration = new HitRateMetrics((int)rateTimeInterval, subInts);
- walFsyncTimeNumber = new HitRateMetrics((int)rateTimeInterval, subInts);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStore.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStore.java
deleted file mode 100755
index 2042358..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStore.java
+++ /dev/null
@@ -1,529 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.file;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.FileChannel;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.store.PageStore;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.IgniteDataIntegrityViolationException;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.PureJavaCrc32;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
-
-/**
- * File page store.
- */
-public class FilePageStore implements PageStore {
- /** Page store file signature. */
- private static final long SIGNATURE = 0xF19AC4FE60C530B8L;
-
- /** File version. */
- private static final int VERSION = 1;
-
- /** Allocated field offset. */
- public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/;
-
- /** */
- private final File cfgFile;
-
- /** */
- private final byte type;
-
- /** Database configuration. */
- private final MemoryConfiguration dbCfg;
-
- /** */
- private RandomAccessFile file;
-
- /** */
- private FileChannel ch;
-
- /** */
- private final AtomicLong allocated;
-
- /** */
- private final int pageSize;
-
- /** */
- private volatile boolean inited;
-
- /** */
- private volatile boolean recover;
-
- /** */
- private volatile int tag;
-
- /** */
- private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
-
- /** */
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
-
- /**
- * @param file File.
- */
- public FilePageStore(byte type, File file, MemoryConfiguration cfg) {
- this.type = type;
-
- cfgFile = file;
- dbCfg = cfg;
-
- allocated = new AtomicLong();
-
- pageSize = dbCfg.getPageSize();
- }
-
- /** {@inheritDoc} */
- @Override public boolean exists() {
- return cfgFile.exists() && cfgFile.length() > HEADER_SIZE;
- }
-
- /**
- * @param type Type.
- * @param pageSize Page size.
- * @return Byte buffer instance.
- */
- public static ByteBuffer header(byte type, int pageSize) {
- ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
-
- hdr.putLong(SIGNATURE);
-
- hdr.putInt(VERSION);
-
- hdr.put(type);
-
- hdr.putInt(pageSize);
-
- hdr.rewind();
-
- return hdr;
- }
-
- /**
- *
- */
- private long initFile() {
- try {
- ByteBuffer hdr = header(type, dbCfg.getPageSize());
-
- while (hdr.remaining() > 0)
- ch.write(hdr);
- }
- catch (IOException e) {
- throw new IgniteException("Check file failed.", e);
- }
-
- //there is 'super' page in every file
- return HEADER_SIZE + dbCfg.getPageSize();
- }
-
- /**
- *
- */
- private long checkFile() throws IgniteCheckedException {
- try {
- ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
-
- while (hdr.remaining() > 0)
- ch.read(hdr);
-
- hdr.rewind();
-
- long signature = hdr.getLong();
-
- if (SIGNATURE != signature)
- throw new IgniteCheckedException("Failed to verify store file (invalid file signature)" +
- " [expectedSignature=" + U.hexLong(SIGNATURE) +
- ", actualSignature=" + U.hexLong(signature) + ']');
-
- int ver = hdr.getInt();
-
- if (VERSION != ver)
- throw new IgniteCheckedException("Failed to verify store file (invalid file version)" +
- " [expectedVersion=" + VERSION +
- ", fileVersion=" + ver + "]");
-
- byte type = hdr.get();
-
- if (this.type != type)
- throw new IgniteCheckedException("Failed to verify store file (invalid file type)" +
- " [expectedFileType=" + this.type +
- ", actualFileType=" + type + "]");
-
- int pageSize = hdr.getInt();
-
- if (dbCfg.getPageSize() != pageSize)
- throw new IgniteCheckedException("Failed to verify store file (invalid page size)" +
- " [expectedPageSize=" + dbCfg.getPageSize() +
- ", filePageSize=" + pageSize + "]");
-
- long fileSize = file.length();
-
- if (fileSize == HEADER_SIZE) // Every file has a special meta page.
- fileSize = pageSize + HEADER_SIZE;
-
- if ((fileSize - HEADER_SIZE) % pageSize != 0)
- throw new IgniteCheckedException("Failed to verify store file (invalid file size)" +
- " [fileSize=" + U.hexLong(fileSize) +
- ", pageSize=" + U.hexLong(pageSize) + ']');
-
- return fileSize;
- }
- catch (IOException e) {
- throw new IgniteCheckedException("File check failed", e);
- }
- }
-
- /**
- * @param cleanFile {@code True} to delete file.
- * @throws IgniteCheckedException If failed.
- */
- public void stop(boolean cleanFile) throws IgniteCheckedException {
- lock.writeLock().lock();
-
- try {
- if (!inited)
- return;
-
- ch.force(false);
-
- file.close();
-
- if (cleanFile)
- cfgFile.delete();
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- *
- */
- public void truncate(int tag) throws IgniteCheckedException {
- lock.writeLock().lock();
-
- try {
- if (!inited)
- return;
-
- this.tag = tag;
-
- ch.position(0);
-
- file.setLength(0);
-
- allocated.set(initFile());
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- *
- */
- public void beginRecover() {
- lock.writeLock().lock();
-
- try {
- recover = true;
- }
- finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- *
- */
- public void finishRecover() {
- lock.writeLock().lock();
-
- try {
- if (inited)
- allocated.set(ch.size());
-
- recover = false;
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- finally {
- lock.writeLock().unlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void read(long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
- init();
-
- try {
- long off = pageOffset(pageId);
-
- assert pageBuf.capacity() == pageSize;
- assert pageBuf.position() == 0;
- assert pageBuf.order() == ByteOrder.nativeOrder();
-
- int len = pageSize;
-
- do {
- int n = ch.read(pageBuf, off);
-
- // If page was not written yet, nothing to read.
- if (n < 0) {
- pageBuf.put(new byte[pageBuf.remaining()]);
-
- return;
- }
-
- off += n;
-
- len -= n;
- }
- while (len > 0);
-
- int savedCrc32 = PageIO.getCrc(pageBuf);
-
- PageIO.setCrc(pageBuf, 0);
-
- pageBuf.position(0);
-
- if (!skipCrc) {
- int curCrc32 = PureJavaCrc32.calcCrc32(pageBuf, pageSize);
-
- if ((savedCrc32 ^ curCrc32) != 0)
- throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " +
- "[id=" + U.hexLong(pageId) + ", off=" + (off - pageSize) +
- ", file=" + cfgFile.getAbsolutePath() + ", fileSize=" + ch.size() +
- ", savedCrc=" + U.hexInt(savedCrc32) + ", curCrc=" + U.hexInt(curCrc32) + "]");
- }
-
- assert PageIO.getCrc(pageBuf) == 0;
-
- if (keepCrc)
- PageIO.setCrc(pageBuf, savedCrc32);
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Read error", e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public void readHeader(ByteBuffer buf) throws IgniteCheckedException {
- init();
-
- try {
- assert buf.remaining() == HEADER_SIZE;
-
- int len = HEADER_SIZE;
-
- long off = 0;
-
- do {
- int n = ch.read(buf, off);
-
- // If page was not written yet, nothing to read.
- if (n < 0)
- return;
-
- off += n;
-
- len -= n;
- }
- while (len > 0);
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Read error", e);
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed to initialize store file.
- */
- private void init() throws IgniteCheckedException {
- if (!inited) {
- lock.writeLock().lock();
-
- try {
- if (!inited) {
- RandomAccessFile rndFile = null;
-
- IgniteCheckedException err = null;
-
- try {
- file = rndFile = new RandomAccessFile(cfgFile, "rw");
-
- ch = file.getChannel();
-
- if (file.length() == 0)
- allocated.set(initFile());
- else
- allocated.set(checkFile());
-
- inited = true;
- }
- catch (IOException e) {
- throw err = new IgniteCheckedException("Can't open file: " + cfgFile.getName(), e);
- }
- finally {
- if (err != null && rndFile != null)
- try {
- rndFile.close();
- }
- catch (IOException e) {
- err.addSuppressed(e);
- }
- }
- }
- }
- finally {
- lock.writeLock().unlock();
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void write(long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
- init();
-
- lock.readLock().lock();
-
- try {
- if (tag < this.tag)
- return;
-
- long off = pageOffset(pageId);
-
- assert (off >= 0 && off + pageSize <= allocated.get() + HEADER_SIZE) || recover :
- "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
-
- assert pageBuf.capacity() == pageSize;
- assert pageBuf.position() == 0;
- assert pageBuf.order() == ByteOrder.nativeOrder();
- assert PageIO.getCrc(pageBuf) == 0 : U.hexLong(pageId);
-
- int crc32 = skipCrc ? 0 : PureJavaCrc32.calcCrc32(pageBuf, pageSize);
-
- PageIO.setCrc(pageBuf, crc32);
-
- pageBuf.position(0);
-
- int len = pageSize;
-
- do {
- int n = ch.write(pageBuf, off);
-
- off += n;
-
- len -= n;
- }
- while (len > 0);
-
- PageIO.setCrc(pageBuf, 0);
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to write the page to the file store [pageId=" + pageId +
- ", file=" + cfgFile.getAbsolutePath() + ']', e);
- }
- finally {
- lock.readLock().unlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public long pageOffset(long pageId) {
- return (long) PageIdUtils.pageIndex(pageId) * pageSize + HEADER_SIZE;
- }
-
- /** {@inheritDoc} */
- @Override public void sync() throws IgniteCheckedException {
- lock.writeLock().lock();
-
- try {
- init();
-
- ch.force(false);
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Sync error", e);
- }
- finally {
- lock.writeLock().unlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void ensure() throws IgniteCheckedException {
- init();
- }
-
- /** {@inheritDoc} */
- @Override public long allocatePage() throws IgniteCheckedException {
- init();
-
- long off = allocPage();
-
- return off / pageSize;
- }
-
- /**
- *
- */
- private long allocPage() {
- long off;
-
- do {
- off = allocated.get();
-
- if (allocated.compareAndSet(off, off + pageSize))
- break;
- }
- while (true);
-
- return off;
- }
-
- /** {@inheritDoc} */
- @Override public int pages() {
- if (!inited)
- return 0;
-
- return (int)(allocated.get() / pageSize);
- }
-}
[22/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalDirectoriesConfigurationTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalDirectoriesConfigurationTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalDirectoriesConfigurationTest.java
new file mode 100644
index 0000000..97e70c2
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalDirectoriesConfigurationTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteWalDirectoriesConfigurationTest extends GridCommonAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ pCfg.setWalStorePath("test/db/wal");
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ return cfg;
+ }
+
+ /**
+ *
+ */
+ public void testPartialWalConfigurationNotAllowed() {
+ try {
+ startGrid();
+ }
+ catch (Exception ignore) {
+ return;
+ }
+
+ fail("Node successfully started with incorrect configuration, exception is expected.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
new file mode 100644
index 0000000..5f92100
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalHistoryReservationsTest.java
@@ -0,0 +1,395 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
+
+/**
+ *
+ */
+public class IgniteWalHistoryReservationsTest extends GridCommonAbstractTest {
+ /** */
+ private volatile boolean client;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setClientMode(client);
+
+ MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ memCfg.setMemoryPolicies(new MemoryPolicyConfiguration().setInitialSize(200 * 1024 * 1024)
+ .setMaxSize(200 * 1024 * 1024).setName("dfltMemPlc"));
+
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(memCfg);
+
+ CacheConfiguration ccfg1 = new CacheConfiguration();
+
+ ccfg1.setName("cache1");
+ ccfg1.setBackups(1);
+ ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ cfg.setCacheConfiguration(ccfg1);
+
+ cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ System.clearProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD);
+
+ client = false;
+
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testReservedOnExchange() throws Exception {
+ System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+ final int entryCnt = 10_000;
+ final int initGridCnt = 4;
+
+ final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt);
+
+ IgniteCache<Object, Object> cache = ig0.cache("cache1");
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, k);
+
+ forceCheckpoint();
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, k * 2);
+
+ forceCheckpoint();
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, k);
+
+ forceCheckpoint();
+
+ Lock lock = cache.lock(0);
+
+ lock.lock();
+
+ try {
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ startGrid(initGridCnt);
+ }
+ catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+ });
+
+ boolean reserved = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ for (int g = 0; g < initGridCnt; g++) {
+ IgniteEx ig = grid(g);
+
+ FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
+
+ Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
+
+ synchronized (archiver) {
+ Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
+
+ if (reserved.isEmpty())
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }, 10_000);
+
+ assert reserved;
+ }
+ finally {
+ lock.unlock();
+ }
+
+ boolean released = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ for (int g = 0; g < initGridCnt; g++) {
+ IgniteEx ig = grid(g);
+
+ FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
+
+ Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
+
+ synchronized (archiver) {
+ Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
+
+ if (!reserved.isEmpty())
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }, 10_000);
+
+ assert released;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRemovesArePreloadedIfHistoryIsAvailable() throws Exception {
+ System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+ int entryCnt = 10_000;
+
+ Ignite ig0 = startGrids(2);
+
+ IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, k);
+
+ forceCheckpoint();
+
+ stopGrid(1);
+
+ for (int k = 0; k < entryCnt; k += 2)
+ cache.remove(k);
+
+ forceCheckpoint();
+
+ Ignite ig1 = startGrid(1);
+
+ IgniteCache<Integer, Integer> cache1 = ig1.cache("cache1");
+
+ assertEquals(entryCnt / 2, cache.size());
+ assertEquals(entryCnt / 2, cache1.size());
+
+ for (Integer k = 0; k < entryCnt; k++) {
+ if (k % 2 == 0) {
+ assertTrue("k=" + k, !cache.containsKey(k));
+ assertTrue("k=" + k, !cache1.containsKey(k));
+ }
+ else {
+ assertEquals("k=" + k, k, cache.get(k));
+ assertEquals("k=" + k, k, cache1.get(k));
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeIsClearedIfHistoryIsUnavailable() throws Exception {
+ int entryCnt = 10_000;
+
+ Ignite ig0 = startGrids(2);
+
+ IgniteCache<Integer, Integer> cache = ig0.cache("cache1");
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, k);
+
+ forceCheckpoint();
+
+ stopGrid(1);
+
+ for (int k = 0; k < entryCnt; k += 2)
+ cache.remove(k);
+
+ forceCheckpoint();
+
+ for (Integer k = 0; k < entryCnt; k++) {
+ if (k % 2 == 0)
+ assertTrue("k=" + k, !cache.containsKey(k));
+ else
+ assertEquals("k=" + k, k, cache.get(k));
+ }
+
+ Ignite ig1 = startGrid(1);
+
+ IgniteCache<Integer, Integer> cache1 = ig1.cache("cache1");
+
+ assertEquals(entryCnt / 2, cache.size());
+ assertEquals(entryCnt / 2, cache1.size());
+
+ for (Integer k = 0; k < entryCnt; k++) {
+ if (k % 2 == 0) {
+ assertTrue("k=" + k, !cache.containsKey(k));
+ assertTrue("k=" + k, !cache1.containsKey(k));
+ }
+ else {
+ assertEquals("k=" + k, k, cache.get(k));
+ assertEquals("k=" + k, k, cache1.get(k));
+ }
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testNodeLeftDuringExchange() throws Exception {
+ System.setProperty(IGNITE_PDS_WAL_REBALANCE_THRESHOLD, "0");
+
+ final int entryCnt = 10_000;
+ final int initGridCnt = 4;
+
+ final IgniteEx ig0 = (IgniteEx)startGrids(initGridCnt);
+
+ IgniteCache<Object, Object> cache = ig0.cache("cache1");
+
+ for (int k = 0; k < entryCnt; k++)
+ cache.put(k, k);
+
+ forceCheckpoint();
+
+ Lock lock = cache.lock(0);
+
+ lock.lock();
+
+ try {
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ startGrid(initGridCnt);
+ }
+ catch (Exception e) {
+ fail(e.getMessage());
+ }
+ }
+ });
+
+ boolean reserved = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ for (int g = 0; g < initGridCnt; g++) {
+ IgniteEx ig = grid(g);
+
+ FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
+
+ Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
+
+ synchronized (archiver) {
+ Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
+
+ if (reserved.isEmpty())
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }, 10_000);
+
+ assert reserved;
+
+ stopGrid(Integer.toString(initGridCnt - 1), true, false);
+ }
+ finally {
+ lock.unlock();
+ }
+
+ boolean released = GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ for (int g = 0; g < initGridCnt - 1; g++) {
+ IgniteEx ig = grid(g);
+
+ FileWriteAheadLogManager wal = (FileWriteAheadLogManager)ig.context().cache().context().wal();
+
+ Object archiver = GridTestUtils.getFieldValue(wal, "archiver");
+
+ synchronized (archiver) {
+ Map reserved = GridTestUtils.getFieldValue(archiver, "reserved");
+
+ if (!reserved.isEmpty())
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }, 10_000);
+
+ assert released;
+
+ awaitPartitionMapExchange();
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ *
+ */
+ private void forceCheckpoint() throws Exception {
+ for (Ignite ignite : G.allGrids()) {
+ if (ignite.cluster().localNode().isClient())
+ continue;
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.waitForCheckpoint("test");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoverySeveralRestartsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoverySeveralRestartsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoverySeveralRestartsTest.java
new file mode 100644
index 0000000..99b7383
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoverySeveralRestartsTest.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.util.Random;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class IgniteWalRecoverySeveralRestartsTest extends GridCommonAbstractTest {
+ /** */
+ public static final int PAGE_SIZE = 1024;
+
+ /** */
+ private static final int KEYS_COUNT = 100_000;
+
+ /** */
+ private static final int LARGE_KEYS_COUNT = 5_000;
+
+ /** */
+ private static final Random rnd = new Random(System.currentTimeMillis());
+
+ /** */
+ private String cacheName = "test";
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 3600_000;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(cacheName);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
+ ccfg.setIndexedTypes(Integer.class, IndexedObject.class);
+ ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
+ ccfg.setReadFromBackup(true);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setPageSize(PAGE_SIZE);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(500 * 1024 * 1024);
+ memPlcCfg.setMaxSize(500 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ binCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ super.beforeTest();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRecoverySeveralRestarts() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ Random locRandom = ThreadLocalRandom.current();
+
+ try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer(cacheName)) {
+ for (int i = 0; i < KEYS_COUNT; ++i) {
+ if (i % (KEYS_COUNT / 100) == 0)
+ info("Loading " + i * 100 / KEYS_COUNT + "%");
+
+ dataLdr.addData(i, new IndexedObject(i));
+ }
+ }
+
+ int size = ignite.cache(cacheName).size();
+
+ for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
+ stopGrid(1, true);
+
+ info("Restart #" + restartCnt);
+ U.sleep(500);
+
+ ignite = startGrid(1);
+
+ IgniteCache<Integer, IndexedObject> cache = ignite.cache(cacheName);
+
+ assertEquals(size, cache.size());
+
+ info("Restart #" + restartCnt);
+
+ for (int i = 0; i < KEYS_COUNT / 100; ++i) {
+ assertNotNull(cache.get(locRandom.nextInt(KEYS_COUNT / 100)));
+
+ cache.put(locRandom.nextInt(KEYS_COUNT / 100), new IndexedObject(locRandom.nextInt(KEYS_COUNT / 100)));
+ }
+
+ cache.put(KEYS_COUNT + restartCnt, new IndexedObject(KEYS_COUNT + restartCnt));
+
+ // Check recovery for partition meta pages.
+ size = cache.size();
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRecoveryWithDynamicCache() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ CacheConfiguration<Integer, IndexedObject> dynCacheCfg = new CacheConfiguration<>();
+
+ dynCacheCfg.setName("dyncache");
+ dynCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ dynCacheCfg.setRebalanceMode(CacheRebalanceMode.NONE);
+ dynCacheCfg.setIndexedTypes(Integer.class, IndexedObject.class);
+ dynCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dynCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
+ dynCacheCfg.setReadFromBackup(true);
+
+ ignite.getOrCreateCache(dynCacheCfg);
+
+ try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer("dyncache")) {
+ for (int i = 0; i < KEYS_COUNT; ++i) {
+ if (i % (KEYS_COUNT / 100) == 0)
+ info("Loading " + i * 100 / KEYS_COUNT + "%");
+
+ dataLdr.addData(i, new IndexedObject(i));
+ }
+ }
+
+ for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
+ stopGrid(1, true);
+
+ info("Restart #" + restartCnt);
+ U.sleep(500);
+
+ ignite = startGrid(1);
+
+ ThreadLocalRandom locRandom = ThreadLocalRandom.current();
+
+ IgniteCache<Integer, IndexedObject> cache = ignite.getOrCreateCache(dynCacheCfg);
+
+ for (int i = 0; i < KEYS_COUNT; ++i)
+ assertNotNull(cache.get(locRandom.nextInt(KEYS_COUNT)));
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRecoveryWithDynamicCacheLargeObjects() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ CacheConfiguration<Integer, IndexedObject> dynCacheCfg = new CacheConfiguration<>();
+
+ dynCacheCfg.setName("dyncache");
+ dynCacheCfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ dynCacheCfg.setRebalanceMode(CacheRebalanceMode.NONE);
+ dynCacheCfg.setIndexedTypes(Integer.class, IndexedObject.class);
+ dynCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dynCacheCfg.setAffinity(new RendezvousAffinityFunction(false, 64 * 4)); // 64 per node
+ dynCacheCfg.setReadFromBackup(true);
+
+ ignite.getOrCreateCache(dynCacheCfg);
+
+ try (IgniteDataStreamer<Integer, IndexedObject> dataLdr = ignite.dataStreamer("dyncache")) {
+ for (int i = 0; i < LARGE_KEYS_COUNT; ++i) {
+ if (i % (LARGE_KEYS_COUNT / 100) == 0)
+ info("Loading " + i * 100 / LARGE_KEYS_COUNT + "%");
+
+ IndexedObject obj = new IndexedObject(i);
+
+ obj.payload = new byte[PAGE_SIZE + 2];
+
+ dataLdr.addData(i, obj);
+ }
+ }
+
+ for (int restartCnt = 0; restartCnt < 5; ++restartCnt) {
+ stopGrid(1, true);
+
+ info("Restart #" + restartCnt);
+ U.sleep(500);
+
+ ignite = startGrid(1);
+
+ ThreadLocalRandom locRandom = ThreadLocalRandom.current();
+
+ IgniteCache<Integer, IndexedObject> cache = ignite.getOrCreateCache(dynCacheCfg);
+
+ for (int i = 0; i < LARGE_KEYS_COUNT; ++i) {
+ IndexedObject val = cache.get(locRandom.nextInt(LARGE_KEYS_COUNT));
+
+ assertNotNull(val);
+
+ assertEquals(PAGE_SIZE + 2, val.payload.length);
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class IndexedObject {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String strVal0;
+
+ /** */
+ @QuerySqlField(index = true)
+ private String strVal1;
+
+ /** */
+ private byte[] payload;
+
+ /**
+ * @param iVal Integer value.
+ */
+ private IndexedObject(int iVal) {
+ this.iVal = iVal;
+
+ strVal0 = "String value #0 " + iVal + " " + GridTestUtils.randomString(rnd, 256);
+ strVal1 = GridTestUtils.randomString(rnd, 256);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ IndexedObject obj = (IndexedObject)o;
+
+ if (iVal != obj.iVal)
+ return false;
+
+ if (strVal0 != null ? !strVal0.equals(obj.strVal0) : obj.strVal0 != null)
+ return false;
+
+ return strVal1 != null ? strVal1.equals(obj.strVal1) : obj.strVal1 == null;
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = iVal;
+
+ res = 31 * res + (strVal0 != null ? strVal0.hashCode() : 0);
+ res = 31 * res + (strVal1 != null ? strVal1.hashCode() : 0);
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IndexedObject.class, this);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
new file mode 100644
index 0000000..f4c732b
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalRecoveryTest.java
@@ -0,0 +1,1326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCompute;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
+import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.PA;
+import org.apache.ignite.internal.util.typedef.PAX;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteCallable;
+import org.apache.ignite.lang.IgnitePredicate;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.resources.IgniteInstanceResource;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy;
+import org.junit.Assert;
+import sun.nio.ch.DirectBuffer;
+
+/**
+ *
+ */
+public class IgniteWalRecoveryTest extends GridCommonAbstractTest {
+ /** */
+ private static final String HAS_CACHE = "HAS_CACHE";
+
+ /** */
+ private static final int LARGE_ARR_SIZE = 1025;
+
+ /** */
+ private boolean fork;
+
+ /** */
+ private String cacheName;
+
+ /** */
+ private int walSegmentSize;
+
+ /** Logger only. */
+ private boolean logOnly;
+
+ /** {@inheritDoc} */
+ @Override protected boolean isMultiJvm() {
+ return fork;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedObject> ccfg = new CacheConfiguration<>(cacheName);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg.setNodeFilter(new RemoteNodeFilter());
+ ccfg.setIndexedTypes(Integer.class, IndexedObject.class);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setPageSize(4 * 1024);
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+
+ memPlcCfg.setName("dfltMemPlc");
+ memPlcCfg.setInitialSize(1024 * 1024 * 1024);
+ memPlcCfg.setMaxSize(1024 * 1024 * 1024);
+
+ dbCfg.setMemoryPolicies(memPlcCfg);
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ if (logOnly)
+ pCfg.setWalMode(WALMode.LOG_ONLY);
+
+ if (walSegmentSize != 0)
+ pCfg.setWalSegmentSize(walSegmentSize);
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ binCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ if (!getTestIgniteInstanceName(0).equals(gridName))
+ cfg.setUserAttributes(F.asMap(HAS_CACHE, true));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ cacheName = "partitioned";
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ logOnly = false;
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalBig() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ Random rnd = new Random();
+
+ Map<Integer, IndexedObject> map = new HashMap<>();
+
+ for (int i = 0; i < 10_000; i++) {
+ if (i % 1000 == 0)
+ X.println(" >> " + i);
+
+ int k = rnd.nextInt(300_000);
+ IndexedObject v = new IndexedObject(rnd.nextInt(10_000));
+
+ cache.put(k, v);
+ map.put(k, v);
+ }
+
+ // Check.
+ for (Integer k : map.keySet())
+ assertEquals(map.get(k), cache.get(k));
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ ignite.active(true);
+
+ cache = ignite.cache("partitioned");
+
+ // Check.
+ for (Integer k : map.keySet())
+ assertEquals(map.get(k), cache.get(k));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testSwitchClassLoader() throws Exception {
+ try {
+ final IgniteEx igniteEx = startGrid(1);
+
+ // CustomDiscoveryMessage will trigger service tasks
+ startGrid(2);
+
+ igniteEx.active(true);
+
+ IgniteCache<Integer, EnumVal> cache = igniteEx.cache("partitioned");
+
+ // Creates LoadCacheJobV2
+// cache.loadCache(null);
+
+ final ClassLoader oldCl = Thread.currentThread().getContextClassLoader();
+ final ClassLoader newCl = getExternalClassLoader();
+
+ Thread.currentThread().setContextClassLoader(newCl);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
+
+ for (int i = 0; i < 10; i++)
+ assert cache.containsKey(i);
+
+ // Invokes ClearTask with new class loader
+ cache.clear();
+
+ Thread.currentThread().setContextClassLoader(oldCl);
+
+ for (int i = 0; i < 10; i++)
+ cache.put(i, i % 2 == 0 ? EnumVal.VAL1 : EnumVal.VAL2);
+
+ for (int i = 0; i < 10; i++)
+ assert cache.containsKey(i);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalSimple() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ info(" --> step1");
+
+ for (int i = 0; i < 10_000; i += 2) {
+// X.println(" -> put: " + i);
+
+ cache.put(i, new IndexedObject(i));
+ }
+
+ info(" --> step2");
+
+ for (int i = 0; i < 10_000; i += 3)
+ cache.put(i, new IndexedObject(i * 2));
+
+ info(" --> step3");
+
+ for (int i = 0; i < 10_000; i += 7)
+ cache.put(i, new IndexedObject(i * 3));
+
+ info(" --> check1");
+
+ // Check.
+ for (int i = 0; i < 10_000; i++) {
+ IndexedObject o;
+
+ if (i % 7 == 0)
+ o = new IndexedObject(i * 3);
+ else if (i % 3 == 0)
+ o = new IndexedObject(i * 2);
+ else if (i % 2 == 0)
+ o = new IndexedObject(i);
+ else
+ o = null;
+
+ assertEquals(o, cache.get(i));
+ }
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ ignite.active(true);
+
+ cache = ignite.cache("partitioned");
+
+ info(" --> check2");
+
+ // Check.
+ for (int i = 0; i < 10_000; i++) {
+ IndexedObject o;
+
+ if (i % 7 == 0)
+ o = new IndexedObject(i * 3);
+ else if (i % 3 == 0)
+ o = new IndexedObject(i * 2);
+ else if (i % 2 == 0)
+ o = new IndexedObject(i);
+ else
+ o = null;
+
+ assertEquals(o, cache.get(i));
+ }
+
+ info(" --> ok");
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testWalLargeValue() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ for (int i = 0; i < 10_000; i++) {
+ final byte[] data = new byte[i];
+
+ Arrays.fill(data, (byte)i);
+
+ cache.put(i, data);
+
+ if (i % 1000 == 0)
+ X.println(" ---> put: " + i);
+
+// Assert.assertArrayEquals(data, (byte[])cache.get(i));
+ }
+
+// info(" --> check1");
+//
+// for (int i = 0; i < 25_000; i++) {
+// final byte[] data = new byte[i];
+//
+// Arrays.fill(data, (byte)i);
+//
+// final byte[] loaded = (byte[]) cache.get(i);
+//
+// Assert.assertArrayEquals(data, loaded);
+// }
+
+ stopGrid(1);
+
+ ignite = startGrid(1);
+
+ ignite.active(true);
+
+ cache = ignite.cache("partitioned");
+
+ info(" --> check2");
+
+ for (int i = 0; i < 10_000; i++) {
+ final byte[] data = new byte[i];
+
+ Arrays.fill(data, (byte)i);
+
+ final byte[] loaded = (byte[]) cache.get(i);
+
+ Assert.assertArrayEquals(data, loaded);
+
+ if (i % 1000 == 0)
+ X.println(" ---> get: " + i);
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRolloverMultithreadedDefault() throws Exception {
+ logOnly = false;
+
+ checkWalRolloverMultithreaded();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRolloverMultithreadedLogOnly() throws Exception {
+ logOnly = true;
+
+ checkWalRolloverMultithreaded();
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testHugeCheckpointRecord() throws Exception {
+ try {
+ final IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ for (int i = 0; i < 50; i++) {
+ CacheConfiguration<Object, Object> ccfg = new CacheConfiguration<>("cache-" + i);
+
+ // We can get 'too many open files' with default number of partitions.
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, 128));
+
+ IgniteCache<Object, Object> cache = ignite.getOrCreateCache(ccfg);
+
+ cache.put(i, i);
+ }
+
+ final long endTime = System.currentTimeMillis() + 30_000;
+
+ IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Random rnd = ThreadLocalRandom.current();
+
+ while (U.currentTimeMillis() < endTime) {
+ IgniteCache<Object, Object> cache = ignite.cache("cache-" + rnd.nextInt(50));
+
+ cache.put(rnd.nextInt(50_000), rnd.nextInt());
+ }
+
+ return null;
+ }
+ }, 16, "put-thread");
+
+ while (System.currentTimeMillis() < endTime) {
+ ignite.context().cache().context().database().wakeupForCheckpoint("test").get();
+
+ U.sleep(500);
+ }
+
+ fut.get();
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ private void checkWalRolloverMultithreaded() throws Exception {
+ walSegmentSize = 2 * 1024 * 1024;
+
+ final long endTime = System.currentTimeMillis() + 2 * 60 * 1000;
+
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ final IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Random rnd = ThreadLocalRandom.current();
+
+ while (U.currentTimeMillis() < endTime)
+ cache.put(rnd.nextInt(50_000), rnd.nextInt());
+
+ return null;
+ }
+ }, 16, "put-thread");
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testWalRenameDirSimple() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ for (int i = 0; i < 100; i++)
+ cache.put(i, new IndexedObject(i));
+
+ stopGrid(1);
+
+ final File cacheDir = cacheDir("partitioned", ignite.context().discovery().consistentId().toString());
+
+ final boolean renamed = cacheDir.renameTo(new File(cacheDir.getParent(), "cache-partitioned0"));
+
+ assert renamed;
+
+ cacheName = "partitioned0";
+
+ ignite = startGrid(1);
+
+ ignite.active(true);
+
+ cache = ignite.cache(cacheName);
+
+ for (int i = 0; i < 100; i++)
+ assertEquals(new IndexedObject(i), cache.get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param cacheName Cache name.
+ * @param consId Consistent ID.
+ * @return Cache dir.
+ * @throws IgniteCheckedException If fail.
+ */
+ private File cacheDir(final String cacheName, String consId) throws IgniteCheckedException {
+ consId = consId.replaceAll("[\\.:]", "_");
+
+ final File dbDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false);
+
+ assert dbDir.exists();
+
+ final File consIdDir = new File(dbDir.getAbsolutePath(), consId);
+
+ assert consIdDir.exists();
+
+ final File cacheDir = new File(consIdDir.getAbsolutePath(), "cache-" + cacheName);
+
+ assert cacheDir.exists();
+
+ return cacheDir;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRecoveryNoCheckpoint() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.active(true);
+
+ ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LoadRunnable(false));
+
+ info("Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ final IgniteEx g0 = ctrlGrid;
+
+ GridTestUtils.waitForCondition(new PA() {
+ /** {@inheritDoc} */
+ @Override public boolean apply() {
+ return g0.cluster().nodes().size() == 1;
+ }
+ }, getTestTimeout());
+
+ fork = false;
+
+ // Now start the grid and verify that updates were restored from WAL.
+ cacheGrid = startGrid(1);
+
+ IgniteCache<Object, Object> cache = cacheGrid.cache("partitioned");
+
+ for (int i = 0; i < 10_000; i++)
+ assertEquals(new IndexedObject(i), cache.get(i));
+
+ List<List<?>> res = cache.query(new SqlFieldsQuery("select count(iVal) from IndexedObject")).getAll();
+
+ assertEquals(1, res.size());
+ assertEquals(10_000L, res.get(0).get(0));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRecoveryLargeNoCheckpoint() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.active(true);
+
+ ctrlGrid.compute(ctrlGrid.cluster().forRemotes()).run(new LargeLoadRunnable(false));
+
+ info("Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ final IgniteEx g0 = ctrlGrid;
+
+ GridTestUtils.waitForCondition(new PA() {
+ /** {@inheritDoc} */
+ @Override public boolean apply() {
+ return g0.cluster().nodes().size() == 1;
+ }
+ }, getTestTimeout());
+
+ fork = false;
+
+ // Now start the grid and verify that updates were restored from WAL.
+ cacheGrid = startGrid(1);
+
+ IgniteCache<Object, Object> cache = cacheGrid.cache("partitioned");
+
+ for (int i = 0; i < 1000; i++) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ Arrays.fill(data, i);
+
+ final long[] loaded = (long[]) cache.get(i);
+
+ Assert.assertArrayEquals(data, loaded);
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return TimeUnit.MINUTES.toMillis(20);
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRandomCrash() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.active(true);
+
+ IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
+
+ rmt.run(new LoadRunnable(false));
+
+ info(">>> Finished cache population.");
+
+ rmt.run(new AsyncLoadRunnable());
+
+ Thread.sleep(20_000);
+
+ info(">>> Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ startGrid(1);
+
+ Boolean res = rmt.call(new VerifyCallable());
+
+ assertTrue(res);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testLargeRandomCrash() throws Exception {
+ try {
+ IgniteEx ctrlGrid = startGrid(0);
+
+ fork = true;
+
+ IgniteEx cacheGrid = startGrid(1);
+
+ ctrlGrid.active(true);
+
+ IgniteCompute rmt = ctrlGrid.compute(ctrlGrid.cluster().forRemotes());
+
+ rmt.run(new LargeLoadRunnable(false));
+
+ info(">>> Finished cache population.");
+
+ rmt.run(new AsyncLargeLoadRunnable());
+
+ Thread.sleep(20_000);
+
+ info(">>> Killing remote process...");
+
+ ((IgniteProcessProxy)cacheGrid).kill();
+
+ startGrid(1);
+
+ Boolean res = rmt.call(new VerifyLargeCallable());
+
+ assertTrue(res);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class RemoteNodeFilter implements IgnitePredicate<ClusterNode> {
+ /** {@inheritDoc} */
+ @Override public boolean apply(ClusterNode clusterNode) {
+ return clusterNode.attribute(HAS_CACHE) != null;
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testDestroyCache() throws Exception {
+ try {
+ IgniteEx ignite = startGrid(1);
+
+ ignite.active(true);
+
+ IgniteCache<Object, Object> cache = ignite.getOrCreateCache("test");
+
+ cache.put(1, new IndexedObject(1));
+
+ ignite.destroyCache("test");
+
+ cache = ignite.getOrCreateCache("test");
+
+ // No entry available after cache destroy.
+ assertNull(cache.get(1));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If fail.
+ */
+ public void testEvictPartition() throws Exception {
+ try {
+ Ignite ignite1 = startGrid("node1");
+
+ ignite1.active(true);
+
+ IgniteCache<Object, Object> cache1 = ignite1.cache(cacheName);
+
+ for (int i = 0; i < 100; i++)
+ cache1.put(i, new IndexedObject(i));
+
+ Ignite ignite2 = startGrid("node2");
+
+ IgniteCache<Object, Object> cache2 = ignite2.cache(cacheName);
+
+ for (int i = 0; i < 100; i++) {
+ assertEquals(new IndexedObject(i), cache1.get(i));
+ assertEquals(new IndexedObject(i), cache2.get(i));
+ }
+
+ ignite1.close();
+ ignite2.close();
+
+ ignite1 = startGrid("node1");
+ ignite2 = startGrid("node2");
+
+ ignite1.active(true);
+
+ cache1 = ignite1.cache(cacheName);
+ cache2 = ignite2.cache(cacheName);
+
+ for (int i = 0; i < 100; i++) {
+ assertEquals(new IndexedObject(i), cache1.get(i));
+ assertEquals(new IndexedObject(i), cache2.get(i));
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testApplyDeltaRecords() throws Exception {
+ try {
+ IgniteEx ignite0 = (IgniteEx)startGrid("node0");
+
+ ignite0.active(true);
+
+ IgniteCache<Object, Object> cache0 = ignite0.cache(cacheName);
+
+ for (int i = 0; i < 1000; i++)
+ cache0.put(i, new IndexedObject(i));
+
+ GridCacheSharedContext<Object, Object> sharedCtx = ignite0.context().cache().context();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)sharedCtx.database();
+
+ db.waitForCheckpoint("test");
+ db.enableCheckpoints(false).get();
+
+ // Log something to know where to start.
+ WALPointer ptr = sharedCtx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
+
+ info("Replay marker: " + ptr);
+
+ for (int i = 1000; i < 5000; i++)
+ cache0.put(i, new IndexedObject(i));
+
+ info("Done puts...");
+
+ for (int i = 2_000; i < 3_000; i++)
+ cache0.remove(i);
+
+ info("Done removes...");
+
+ for (int i = 5000; i < 6000; i++)
+ cache0.put(i, new IndexedObject(i));
+
+ info("Done puts...");
+
+ Map<FullPageId, byte[]> rolledPages = new HashMap<>();
+
+ int pageSize = sharedCtx.database().pageSize();
+
+ ByteBuffer buf1 = ByteBuffer.allocateDirect(pageSize);
+
+ // Now check that deltas can be correctly applied.
+ try (WALIterator it = sharedCtx.wal().replay(ptr)) {
+ while (it.hasNext()) {
+ IgniteBiTuple<WALPointer, WALRecord> tup = it.next();
+
+ WALRecord rec = tup.get2();
+
+ if (rec instanceof PageSnapshot) {
+ PageSnapshot page = (PageSnapshot)rec;
+
+ rolledPages.put(page.fullPageId(), page.pageData());
+ }
+ else if (rec instanceof PageDeltaRecord) {
+ PageDeltaRecord delta = (PageDeltaRecord)rec;
+
+ FullPageId fullId = new FullPageId(delta.pageId(), delta.cacheId());
+
+ byte[] pageData = rolledPages.get(fullId);
+
+ if (pageData == null) {
+ pageData = new byte[pageSize];
+
+ rolledPages.put(fullId, pageData);
+ }
+
+ assertNotNull("Missing page snapshot [page=" + fullId + ", delta=" + delta + ']', pageData);
+
+ buf1.order(ByteOrder.nativeOrder());
+
+ buf1.position(0);
+ buf1.put(pageData);
+ buf1.position(0);
+
+ delta.applyDelta(sharedCtx
+ .database()
+ .memoryPolicy(null)
+ .pageMemory(),
+
+ ((DirectBuffer)buf1).address());
+
+ buf1.position(0);
+
+ buf1.get(pageData);
+ }
+ }
+ }
+
+ info("Done apply...");
+
+ PageMemoryEx pageMem = (PageMemoryEx)db.memoryPolicy(null).pageMemory();
+
+ for (Map.Entry<FullPageId, byte[]> entry : rolledPages.entrySet()) {
+ FullPageId fullId = entry.getKey();
+
+ ignite0.context().cache().context().database().checkpointReadLock();
+
+ try {
+ long page = pageMem.acquirePage(fullId.cacheId(), fullId.pageId(), true);
+
+ try {
+ long buf = pageMem.writeLock(fullId.cacheId(), fullId.pageId(), page, true);
+
+ try {
+ byte[] data = entry.getValue();
+
+ for (int i = 0; i < data.length; i++) {
+ if (fullId.pageId() == TrackingPageIO.VERSIONS.latest().trackingPageFor(fullId.pageId(), db.pageSize()))
+ continue; // Skip tracking pages.
+
+ assertEquals("page=" + fullId + ", pos=" + i, PageUtils.getByte(buf, i), data[i]);
+ }
+ }
+ finally {
+ pageMem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, false, true);
+ }
+ }
+ finally {
+ pageMem.releasePage(fullId.cacheId(), fullId.pageId(), page);
+ }
+ }
+ finally {
+ ignite0.context().cache().context().database().checkpointReadUnlock();
+ }
+ }
+
+ ignite0.close();
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class LoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** */
+ private boolean disableCheckpoints;
+
+ /**
+ * @param disableCheckpoints Disable checkpoints flag.
+ */
+ private LoadRunnable(boolean disableCheckpoints) {
+ this.disableCheckpoints = disableCheckpoints;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ ignite.log().info("Started load.");
+
+ if (disableCheckpoints) {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ try {
+ dbMgr.enableCheckpoints(false).get();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ }
+
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache("partitioned") != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ for (int i = 0; i < 10_000; i++)
+ cache.put(i, new IndexedObject(i));
+
+ ignite.log().info("Finished load.");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class AsyncLoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache("partitioned") != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ignite.log().info(">>>>>>> Started load.");
+
+ for (int i = 0; i < 4; i++) {
+ ignite.scheduler().callLocal(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int cnt = 0;
+
+ while (!Thread.currentThread().isInterrupted()) {
+ cache.put(rnd.nextInt(10_000), new IndexedObject(rnd.nextInt()));
+
+ cnt++;
+
+ if (cnt > 0 && cnt % 1_000 == 0)
+ ignite.log().info(">>>> Updated: " + cnt);
+ }
+
+ return null;
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class VerifyCallable implements IgniteCallable<Boolean> {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public Boolean call() throws Exception {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache("partitioned") != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ for (int i = 0; i < 10_000; i++) {
+ Object val = cache.get(i);
+
+ if (val == null) {
+ ignite.log().warning("Failed to find a value for key: " + i);
+
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }
+
+ /**
+ *
+ */
+ private static class LargeLoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** */
+ private boolean disableCheckpoints;
+
+ /**
+ * @param disableCheckpoints Disable checkpoints flag.
+ */
+ private LargeLoadRunnable(boolean disableCheckpoints) {
+ this.disableCheckpoints = disableCheckpoints;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache("partitioned") != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ignite.log().info("Started load.");
+
+ if (disableCheckpoints) {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ for (int i = 0; i < 1000; i++) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ Arrays.fill(data, i);
+
+ cache.put(i, data);
+ }
+
+ ignite.log().info("Finished load.");
+ }
+ }
+
+ /**
+ *
+ */
+ private static class AsyncLargeLoadRunnable implements IgniteRunnable {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public void run() {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache("partitioned") != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ ignite.log().info(">>>>>>> Started load.");
+
+ for (int i = 0; i < 1; i++) {
+ ignite.scheduler().callLocal(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int cnt = 0;
+
+ while (!Thread.currentThread().isInterrupted()) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ final int key = rnd.nextInt(1000);
+
+ Arrays.fill(data, key);
+
+// System.out.println("> " + key);
+
+ cache.put(key, data);
+
+ cnt++;
+
+ if (cnt > 0 && cnt % 1_000 == 0)
+ ignite.log().info(">>>> Updated: " + cnt);
+ }
+
+ return null;
+ }
+ });
+ }
+ }
+ }
+
+ /**
+ *
+ */
+ private static class VerifyLargeCallable implements IgniteCallable<Boolean> {
+ /** */
+ @IgniteInstanceResource
+ private Ignite ignite;
+
+ /** {@inheritDoc} */
+ @Override public Boolean call() throws Exception {
+ try {
+ boolean successfulWaiting = GridTestUtils.waitForCondition(new PAX() {
+ @Override public boolean applyx() {
+ return ignite.cache("partitioned") != null;
+ }
+ }, 10_000);
+
+ assertTrue(successfulWaiting);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ throw new RuntimeException(e);
+ }
+
+ IgniteCache<Object, Object> cache = ignite.cache("partitioned");
+
+ for (int i = 0; i < 1000; i++) {
+ final long[] data = new long[LARGE_ARR_SIZE];
+
+ Arrays.fill(data, i);
+
+ final Object val = cache.get(i);
+
+ if (val == null) {
+ ignite.log().warning("Failed to find a value for key: " + i);
+
+ return false;
+ }
+ }
+
+ return true;
+ }
+ }
+
+
+ /**
+ *
+ */
+ private static class IndexedObject {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /**
+ * @param iVal Integer value.
+ */
+ private IndexedObject(int iVal) {
+ this.iVal = iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof IndexedObject))
+ return false;
+
+ IndexedObject that = (IndexedObject)o;
+
+ return iVal == that.iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ return iVal;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(IndexedObject.class, this);
+ }
+ }
+
+ /**
+ *
+ */
+ private enum EnumVal {
+ /** */
+ VAL1,
+
+ /** */
+ VAL2,
+
+ /** */
+ VAL3
+ }
+}
[36/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java
new file mode 100644
index 0000000..035a91a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/evict/RandomLruPageEvictionTracker.java
@@ -0,0 +1,159 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.ignite.internal.processors.cache.persistence.evict;
+
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.internal.util.typedef.internal.LT;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class RandomLruPageEvictionTracker extends PageAbstractEvictionTracker {
+ /** Evict attempts limit. */
+ private static final int EVICT_ATTEMPTS_LIMIT = 30;
+
+ /** LRU Sample size. */
+ private static final int SAMPLE_SIZE = 5;
+
+ /** Maximum sample search spin count */
+ private static final int SAMPLE_SPIN_LIMIT = SAMPLE_SIZE * 1000;
+
+ /** Logger. */
+ private final IgniteLogger log;
+
+ /** Tracking array ptr. */
+ private long trackingArrPtr;
+
+ /**
+ * @param pageMem Page memory.
+ * @param plcCfg Policy config.
+ * @param sharedCtx Shared context.
+ */
+ public RandomLruPageEvictionTracker(
+ PageMemory pageMem,
+ MemoryPolicyConfiguration plcCfg,
+ GridCacheSharedContext<?, ?> sharedCtx
+ ) {
+ super((PageMemoryNoStoreImpl)pageMem, plcCfg, sharedCtx);
+
+ MemoryConfiguration memCfg = sharedCtx.kernalContext().config().getMemoryConfiguration();
+
+ assert plcCfg.getMaxSize() / memCfg.getPageSize() < Integer.MAX_VALUE;
+
+ log = sharedCtx.logger(getClass());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
+ trackingArrPtr = GridUnsafe.allocateMemory(trackingSize * 4);
+
+ GridUnsafe.setMemory(trackingArrPtr, trackingSize * 4, (byte)0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ GridUnsafe.freeMemory(trackingArrPtr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void touchPage(long pageId) throws IgniteCheckedException {
+ int pageIdx = PageIdUtils.pageIndex(pageId);
+
+ long res = compactTimestamp(U.currentTimeMillis());
+
+ assert res >= 0 && res < Integer.MAX_VALUE;
+
+ GridUnsafe.putIntVolatile(null, trackingArrPtr + trackingIdx(pageIdx) * 4, (int)res);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void evictDataPage() throws IgniteCheckedException {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int evictAttemptsCnt = 0;
+
+ while (evictAttemptsCnt < EVICT_ATTEMPTS_LIMIT) {
+ int lruTrackingIdx = -1;
+
+ int lruCompactTs = Integer.MAX_VALUE;
+
+ int dataPagesCnt = 0;
+
+ int sampleSpinCnt = 0;
+
+ while (dataPagesCnt < SAMPLE_SIZE) {
+ int sampleTrackingIdx = rnd.nextInt(trackingSize);
+
+ int compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4);
+
+ if (compactTs != 0) {
+ // We chose data page with at least one touch.
+ if (compactTs < lruCompactTs) {
+ lruTrackingIdx = sampleTrackingIdx;
+
+ lruCompactTs = compactTs;
+ }
+
+ dataPagesCnt++;
+ }
+
+ sampleSpinCnt++;
+
+ if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) {
+ LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT);
+
+ return;
+ }
+ }
+
+ if (evictDataPage(pageIdx(lruTrackingIdx)))
+ return;
+
+ evictAttemptsCnt++;
+ }
+
+ LT.warn(log, "Too many failed attempts to evict page: " + EVICT_ATTEMPTS_LIMIT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean checkTouch(long pageId) {
+ int trackingIdx = trackingIdx(PageIdUtils.pageIndex(pageId));
+
+ int ts = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 4);
+
+ return ts != 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void forgetPage(long pageId) {
+ int pageIdx = PageIdUtils.pageIndex(pageId);
+
+ GridUnsafe.putIntVolatile(null, trackingArrPtr + trackingIdx(pageIdx) * 4, 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
new file mode 100755
index 0000000..6ddc9fc
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -0,0 +1,529 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.channels.FileChannel;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
+
+/**
+ * File page store.
+ */
+public class FilePageStore implements PageStore {
+ /** Page store file signature. */
+ private static final long SIGNATURE = 0xF19AC4FE60C530B8L;
+
+ /** File version. */
+ private static final int VERSION = 1;
+
+ /** Allocated field offset. */
+ public static final int HEADER_SIZE = 8/*SIGNATURE*/ + 4/*VERSION*/ + 1/*type*/ + 4/*page size*/;
+
+ /** */
+ private final File cfgFile;
+
+ /** */
+ private final byte type;
+
+ /** Database configuration. */
+ private final MemoryConfiguration dbCfg;
+
+ /** */
+ private RandomAccessFile file;
+
+ /** */
+ private FileChannel ch;
+
+ /** */
+ private final AtomicLong allocated;
+
+ /** */
+ private final int pageSize;
+
+ /** */
+ private volatile boolean inited;
+
+ /** */
+ private volatile boolean recover;
+
+ /** */
+ private volatile int tag;
+
+ /** */
+ private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
+
+ /** */
+ private final ReadWriteLock lock = new ReentrantReadWriteLock();
+
+ /**
+ * @param file File.
+ */
+ public FilePageStore(byte type, File file, MemoryConfiguration cfg) {
+ this.type = type;
+
+ cfgFile = file;
+ dbCfg = cfg;
+
+ allocated = new AtomicLong();
+
+ pageSize = dbCfg.getPageSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists() {
+ return cfgFile.exists() && cfgFile.length() > HEADER_SIZE;
+ }
+
+ /**
+ * @param type Type.
+ * @param pageSize Page size.
+ * @return Byte buffer instance.
+ */
+ public static ByteBuffer header(byte type, int pageSize) {
+ ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+
+ hdr.putLong(SIGNATURE);
+
+ hdr.putInt(VERSION);
+
+ hdr.put(type);
+
+ hdr.putInt(pageSize);
+
+ hdr.rewind();
+
+ return hdr;
+ }
+
+ /**
+ *
+ */
+ private long initFile() {
+ try {
+ ByteBuffer hdr = header(type, dbCfg.getPageSize());
+
+ while (hdr.remaining() > 0)
+ ch.write(hdr);
+ }
+ catch (IOException e) {
+ throw new IgniteException("Check file failed.", e);
+ }
+
+ //there is 'super' page in every file
+ return HEADER_SIZE + dbCfg.getPageSize();
+ }
+
+ /**
+ *
+ */
+ private long checkFile() throws IgniteCheckedException {
+ try {
+ ByteBuffer hdr = ByteBuffer.allocate(HEADER_SIZE).order(ByteOrder.LITTLE_ENDIAN);
+
+ while (hdr.remaining() > 0)
+ ch.read(hdr);
+
+ hdr.rewind();
+
+ long signature = hdr.getLong();
+
+ if (SIGNATURE != signature)
+ throw new IgniteCheckedException("Failed to verify store file (invalid file signature)" +
+ " [expectedSignature=" + U.hexLong(SIGNATURE) +
+ ", actualSignature=" + U.hexLong(signature) + ']');
+
+ int ver = hdr.getInt();
+
+ if (VERSION != ver)
+ throw new IgniteCheckedException("Failed to verify store file (invalid file version)" +
+ " [expectedVersion=" + VERSION +
+ ", fileVersion=" + ver + "]");
+
+ byte type = hdr.get();
+
+ if (this.type != type)
+ throw new IgniteCheckedException("Failed to verify store file (invalid file type)" +
+ " [expectedFileType=" + this.type +
+ ", actualFileType=" + type + "]");
+
+ int pageSize = hdr.getInt();
+
+ if (dbCfg.getPageSize() != pageSize)
+ throw new IgniteCheckedException("Failed to verify store file (invalid page size)" +
+ " [expectedPageSize=" + dbCfg.getPageSize() +
+ ", filePageSize=" + pageSize + "]");
+
+ long fileSize = file.length();
+
+ if (fileSize == HEADER_SIZE) // Every file has a special meta page.
+ fileSize = pageSize + HEADER_SIZE;
+
+ if ((fileSize - HEADER_SIZE) % pageSize != 0)
+ throw new IgniteCheckedException("Failed to verify store file (invalid file size)" +
+ " [fileSize=" + U.hexLong(fileSize) +
+ ", pageSize=" + U.hexLong(pageSize) + ']');
+
+ return fileSize;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("File check failed", e);
+ }
+ }
+
+ /**
+ * @param cleanFile {@code True} to delete file.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void stop(boolean cleanFile) throws IgniteCheckedException {
+ lock.writeLock().lock();
+
+ try {
+ if (!inited)
+ return;
+
+ ch.force(false);
+
+ file.close();
+
+ if (cleanFile)
+ cfgFile.delete();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ *
+ */
+ public void truncate(int tag) throws IgniteCheckedException {
+ lock.writeLock().lock();
+
+ try {
+ if (!inited)
+ return;
+
+ this.tag = tag;
+
+ ch.position(0);
+
+ file.setLength(0);
+
+ allocated.set(initFile());
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ *
+ */
+ public void beginRecover() {
+ lock.writeLock().lock();
+
+ try {
+ recover = true;
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /**
+ *
+ */
+ public void finishRecover() {
+ lock.writeLock().lock();
+
+ try {
+ if (inited)
+ allocated.set(ch.size());
+
+ recover = false;
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void read(long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
+ init();
+
+ try {
+ long off = pageOffset(pageId);
+
+ assert pageBuf.capacity() == pageSize;
+ assert pageBuf.position() == 0;
+ assert pageBuf.order() == ByteOrder.nativeOrder();
+
+ int len = pageSize;
+
+ do {
+ int n = ch.read(pageBuf, off);
+
+ // If page was not written yet, nothing to read.
+ if (n < 0) {
+ pageBuf.put(new byte[pageBuf.remaining()]);
+
+ return;
+ }
+
+ off += n;
+
+ len -= n;
+ }
+ while (len > 0);
+
+ int savedCrc32 = PageIO.getCrc(pageBuf);
+
+ PageIO.setCrc(pageBuf, 0);
+
+ pageBuf.position(0);
+
+ if (!skipCrc) {
+ int curCrc32 = PureJavaCrc32.calcCrc32(pageBuf, pageSize);
+
+ if ((savedCrc32 ^ curCrc32) != 0)
+ throw new IgniteDataIntegrityViolationException("Failed to read page (CRC validation failed) " +
+ "[id=" + U.hexLong(pageId) + ", off=" + (off - pageSize) +
+ ", file=" + cfgFile.getAbsolutePath() + ", fileSize=" + ch.size() +
+ ", savedCrc=" + U.hexInt(savedCrc32) + ", curCrc=" + U.hexInt(curCrc32) + "]");
+ }
+
+ assert PageIO.getCrc(pageBuf) == 0;
+
+ if (keepCrc)
+ PageIO.setCrc(pageBuf, savedCrc32);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Read error", e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readHeader(ByteBuffer buf) throws IgniteCheckedException {
+ init();
+
+ try {
+ assert buf.remaining() == HEADER_SIZE;
+
+ int len = HEADER_SIZE;
+
+ long off = 0;
+
+ do {
+ int n = ch.read(buf, off);
+
+ // If page was not written yet, nothing to read.
+ if (n < 0)
+ return;
+
+ off += n;
+
+ len -= n;
+ }
+ while (len > 0);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Read error", e);
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed to initialize store file.
+ */
+ private void init() throws IgniteCheckedException {
+ if (!inited) {
+ lock.writeLock().lock();
+
+ try {
+ if (!inited) {
+ RandomAccessFile rndFile = null;
+
+ IgniteCheckedException err = null;
+
+ try {
+ file = rndFile = new RandomAccessFile(cfgFile, "rw");
+
+ ch = file.getChannel();
+
+ if (file.length() == 0)
+ allocated.set(initFile());
+ else
+ allocated.set(checkFile());
+
+ inited = true;
+ }
+ catch (IOException e) {
+ throw err = new IgniteCheckedException("Can't open file: " + cfgFile.getName(), e);
+ }
+ finally {
+ if (err != null && rndFile != null)
+ try {
+ rndFile.close();
+ }
+ catch (IOException e) {
+ err.addSuppressed(e);
+ }
+ }
+ }
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
+ init();
+
+ lock.readLock().lock();
+
+ try {
+ if (tag < this.tag)
+ return;
+
+ long off = pageOffset(pageId);
+
+ assert (off >= 0 && off + pageSize <= allocated.get() + HEADER_SIZE) || recover :
+ "off=" + U.hexLong(off) + ", allocated=" + U.hexLong(allocated.get()) + ", pageId=" + U.hexLong(pageId);
+
+ assert pageBuf.capacity() == pageSize;
+ assert pageBuf.position() == 0;
+ assert pageBuf.order() == ByteOrder.nativeOrder();
+ assert PageIO.getCrc(pageBuf) == 0 : U.hexLong(pageId);
+
+ int crc32 = skipCrc ? 0 : PureJavaCrc32.calcCrc32(pageBuf, pageSize);
+
+ PageIO.setCrc(pageBuf, crc32);
+
+ pageBuf.position(0);
+
+ int len = pageSize;
+
+ do {
+ int n = ch.write(pageBuf, off);
+
+ off += n;
+
+ len -= n;
+ }
+ while (len > 0);
+
+ PageIO.setCrc(pageBuf, 0);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to write the page to the file store [pageId=" + pageId +
+ ", file=" + cfgFile.getAbsolutePath() + ']', e);
+ }
+ finally {
+ lock.readLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long pageOffset(long pageId) {
+ return (long) PageIdUtils.pageIndex(pageId) * pageSize + HEADER_SIZE;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sync() throws IgniteCheckedException {
+ lock.writeLock().lock();
+
+ try {
+ init();
+
+ ch.force(false);
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Sync error", e);
+ }
+ finally {
+ lock.writeLock().unlock();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public synchronized void ensure() throws IgniteCheckedException {
+ init();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long allocatePage() throws IgniteCheckedException {
+ init();
+
+ long off = allocPage();
+
+ return off / pageSize;
+ }
+
+ /**
+ *
+ */
+ private long allocPage() {
+ long off;
+
+ do {
+ off = allocated.get();
+
+ if (allocated.compareAndSet(off, off + pageSize))
+ break;
+ }
+ while (true);
+
+ return off;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int pages() {
+ if (!inited)
+ return 0;
+
+ return (int)(allocated.get() / pageSize);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
new file mode 100755
index 0000000..f908512
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -0,0 +1,671 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.cache.CacheGroupContext;
+import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
+import org.apache.ignite.internal.processors.cache.StoredCacheData;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.marshaller.Marshaller;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * File page store manager.
+ */
+public class FilePageStoreManager extends GridCacheSharedManagerAdapter implements IgnitePageStoreManager {
+ /** File suffix. */
+ public static final String FILE_SUFFIX = ".bin";
+
+ /** Partition file prefix. */
+ public static final String PART_FILE_PREFIX = "part-";
+
+ /** */
+ public static final String INDEX_FILE_NAME = "index" + FILE_SUFFIX;
+
+ /** */
+ public static final String PART_FILE_TEMPLATE = PART_FILE_PREFIX+ "%d" + FILE_SUFFIX;
+
+ /** */
+ public static final String CACHE_DIR_PREFIX = "cache-";
+
+ /** */
+ public static final String CACHE_GRP_DIR_PREFIX = "cacheGroup-";
+
+ /** */
+ public static final String CACHE_DATA_FILENAME = "cache_data.dat";
+
+ /** Marshaller. */
+ private static final Marshaller marshaller = new JdkMarshaller();
+
+ /** */
+ private final Map<Integer, CacheStoreHolder> idxCacheStores = new ConcurrentHashMap<>();
+
+ /** */
+ private final IgniteConfiguration igniteCfg;
+
+ /** */
+ private PersistentStoreConfiguration pstCfg;
+
+ /** Absolute directory for file page store */
+ private File storeWorkDir;
+
+ /** */
+ private final long metaPageId = PageIdUtils.pageId(-1, PageMemory.FLAG_IDX, 0);
+
+ /** */
+ private final Set<Integer> grpsWithoutIdx = Collections.newSetFromMap(new ConcurrentHashMap<Integer, Boolean>());
+
+ /**
+ * @param ctx Kernal context.
+ */
+ public FilePageStoreManager(GridKernalContext ctx) {
+ igniteCfg = ctx.config();
+
+ PersistentStoreConfiguration pstCfg = igniteCfg.getPersistentStoreConfiguration();
+
+ assert pstCfg != null : "WAL should not be created if persistence is disabled.";
+
+ this.pstCfg = pstCfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start0() throws IgniteCheckedException {
+ if (cctx.kernalContext().clientNode())
+ return;
+
+ String consId = U.maskForFileName(cctx.kernalContext().discovery().consistentId().toString());
+
+ if (pstCfg.getPersistentStorePath() != null) {
+ File workDir0 = new File(pstCfg.getPersistentStorePath());
+
+ if (!workDir0.isAbsolute())
+ workDir0 = U.resolveWorkDirectory(
+ igniteCfg.getWorkDirectory(),
+ pstCfg.getPersistentStorePath(),
+ false
+ );
+
+ storeWorkDir = new File(workDir0, consId);
+ }
+ else
+ storeWorkDir = new File(U.resolveWorkDirectory(
+ igniteCfg.getWorkDirectory(),
+ "db",
+ false
+ ), consId);
+
+ U.ensureDirectory(storeWorkDir, "page store work directory", log);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop0(boolean cancel) {
+ if (log.isDebugEnabled())
+ log.debug("Stopping page store manager.");
+
+ IgniteCheckedException ex = shutdown(false);
+
+ if (ex != null)
+ U.error(log, "Failed to gracefully stop page store manager", ex);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("Activate page store manager [id=" + cctx.localNodeId() +
+ " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
+
+ start0();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
+ if (log.isDebugEnabled())
+ log.debug("DeActivate page store manager [id=" + cctx.localNodeId() +
+ " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
+
+ stop0(true);
+
+ idxCacheStores.clear();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void beginRecover() {
+ for (CacheStoreHolder holder : idxCacheStores.values()) {
+ holder.idxStore.beginRecover();
+
+ for (FilePageStore partStore : holder.partStores)
+ partStore.beginRecover();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void finishRecover() {
+ for (CacheStoreHolder holder : idxCacheStores.values()) {
+ holder.idxStore.finishRecover();
+
+ for (FilePageStore partStore : holder.partStores)
+ partStore.finishRecover();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData)
+ throws IgniteCheckedException {
+ int grpId = grpDesc.groupId();
+
+ if (!idxCacheStores.containsKey(grpId)) {
+ CacheStoreHolder holder = initForCache(grpDesc, cacheData.config());
+
+ CacheStoreHolder old = idxCacheStores.put(grpId, holder);
+
+ assert old == null : "Non-null old store holder for cache: " + cacheData.config().getName();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void storeCacheData(
+ CacheGroupDescriptor grpDesc,
+ StoredCacheData cacheData
+ ) throws IgniteCheckedException {
+
+ File cacheWorkDir = cacheWorkDirectory(grpDesc, cacheData.config());
+ File file;
+
+ checkAndInitCacheWorkDir(cacheWorkDir);
+
+ assert cacheWorkDir.exists() : "Work directory does not exist: " + cacheWorkDir;
+
+ if (grpDesc.sharedGroup())
+ file = new File(cacheWorkDir, cacheData.config().getName() + CACHE_DATA_FILENAME);
+ else
+ file = new File(cacheWorkDir, CACHE_DATA_FILENAME);
+
+ if (!file.exists() || file.length() == 0) {
+ try {
+ file.createNewFile();
+
+ try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(file))) {
+ marshaller.marshal(cacheData, stream);
+ }
+ }
+ catch (IOException ex) {
+ throw new IgniteCheckedException("Failed to persist cache configuration: " + cacheData.config().getName(), ex);
+ }
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void shutdownForCacheGroup(CacheGroupContext grp, boolean destroy) throws IgniteCheckedException {
+ grpsWithoutIdx.remove(grp.groupId());
+
+ CacheStoreHolder old = idxCacheStores.remove(grp.groupId());
+
+ assert old != null : "Missing cache store holder [cache=" + grp.cacheOrGroupName() +
+ ", locNodeId=" + cctx.localNodeId() + ", gridName=" + cctx.igniteInstanceName() + ']';
+
+ IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null);
+
+ if (ex != null)
+ throw ex;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onPartitionCreated(int grpId, int partId) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onPartitionDestroyed(int grpId, int partId, int tag) throws IgniteCheckedException {
+ assert partId <= PageIdAllocator.MAX_PARTITION_ID;
+
+ PageStore store = getStore(grpId, partId);
+
+ assert store instanceof FilePageStore : store;
+
+ ((FilePageStore)store).truncate(tag);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void read(int cacheId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException {
+ read(cacheId, pageId, pageBuf, false);
+ }
+
+ /**
+ * Will preserve crc in buffer if keepCrc is true.
+ *
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param pageBuf Page buffer.
+ * @param keepCrc Keep CRC flag.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
+ PageStore store = getStore(cacheId, PageIdUtils.partId(pageId));
+
+ store.read(pageId, pageBuf, keepCrc);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(int grpId, int partId) throws IgniteCheckedException {
+ PageStore store = getStore(grpId, partId);
+
+ return store.exists();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readHeader(int cacheId, int partId, ByteBuffer buf) throws IgniteCheckedException {
+ PageStore store = getStore(cacheId, partId);
+
+ store.readHeader(buf);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(int cacheId, long pageId, ByteBuffer pageBuf,int tag) throws IgniteCheckedException {
+ writeInternal(cacheId, pageId, pageBuf, tag);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long pageOffset(int cacheId, long pageId) throws IgniteCheckedException {
+ PageStore store = getStore(cacheId, PageIdUtils.partId(pageId));
+
+ return store.pageOffset(pageId);
+ }
+
+ /**
+ * @param cacheId Cache ID to write.
+ * @param pageId Page ID.
+ * @param pageBuf Page buffer.
+ * @return PageStore to which the page has been written.
+ * @throws IgniteCheckedException If IO error occurred.
+ */
+ public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
+ int partId = PageIdUtils.partId(pageId);
+
+ PageStore store = getStore(cacheId, partId);
+
+ store.write(pageId, pageBuf, tag);
+
+ return store;
+ }
+
+ /**
+ * @param grpDesc Cache group descriptor.
+ * @param ccfg Cache configuration.
+ * @return Cache work directory.
+ */
+ private File cacheWorkDirectory(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) {
+ String dirName;
+
+ if (grpDesc.sharedGroup())
+ dirName = CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
+ else
+ dirName = CACHE_DIR_PREFIX + ccfg.getName();
+
+ return new File(storeWorkDir, dirName);
+ }
+
+ /**
+ * @param grpDesc Cache group descriptor.
+ * @param ccfg Cache configuration.
+ * @return Cache store holder.
+ * @throws IgniteCheckedException If failed.
+ */
+ private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) throws IgniteCheckedException {
+ assert !grpDesc.sharedGroup() || ccfg.getGroupName() != null : ccfg.getName();
+
+ File cacheWorkDir = cacheWorkDirectory(grpDesc, ccfg);
+
+ boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
+
+ File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
+
+ if (dirExisted && !idxFile.exists())
+ grpsWithoutIdx.add(grpDesc.groupId());
+
+ FilePageStore idxStore = new FilePageStore(
+ PageMemory.FLAG_IDX,
+ idxFile,
+ cctx.kernalContext().config().getMemoryConfiguration());
+
+ FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()];
+
+ for (int partId = 0; partId < partStores.length; partId++) {
+ FilePageStore partStore = new FilePageStore(
+ PageMemory.FLAG_DATA,
+ new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)),
+ cctx.kernalContext().config().getMemoryConfiguration()
+ );
+
+ partStores[partId] = partStore;
+ }
+
+ return new CacheStoreHolder(idxStore, partStores);
+ }
+
+ private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteCheckedException {
+ boolean dirExisted = false;
+
+ if (!cacheWorkDir.exists()) {
+ boolean res = cacheWorkDir.mkdirs();
+
+ if (!res)
+ throw new IgniteCheckedException("Failed to initialize cache working directory " +
+ "(failed to create, make sure the work folder has correct permissions): " +
+ cacheWorkDir.getAbsolutePath());
+ }
+ else {
+ if (cacheWorkDir.isFile())
+ throw new IgniteCheckedException("Failed to initialize cache working directory " +
+ "(a file with the same name already exists): " + cacheWorkDir.getAbsolutePath());
+
+ File lockF = new File(cacheWorkDir, IgniteCacheSnapshotManager.SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME);
+
+ if (lockF.exists()) {
+ Path tmp = cacheWorkDir.toPath().getParent().resolve(cacheWorkDir.getName() + ".tmp");
+
+ boolean deleted = U.delete(cacheWorkDir);
+
+ if (Files.exists(tmp) && Files.isDirectory(tmp)) {
+ U.warn(log, "Ignite node crashed during the snapshot restore process " +
+ "(there is a snapshot restore lock file left for cache). But old version of cache was saved. " +
+ "Trying to restore it. Cache - [" + cacheWorkDir.getAbsolutePath() + ']');
+
+ try {
+ Files.move(tmp, cacheWorkDir.toPath());
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+ else {
+ U.warn(log, "Ignite node crashed during the snapshot restore process " +
+ "(there is a snapshot restore lock file left for cache). Will remove both the lock file and " +
+ "incomplete cache directory [cacheDir=" + cacheWorkDir.getAbsolutePath() + ']');
+
+ if (!deleted)
+ throw new IgniteCheckedException("Failed to remove obsolete cache working directory " +
+ "(remove the directory manually and make sure the work folder has correct permissions): " +
+ cacheWorkDir.getAbsolutePath());
+
+ cacheWorkDir.mkdirs();
+ }
+
+ if (!cacheWorkDir.exists())
+ throw new IgniteCheckedException("Failed to initialize cache working directory " +
+ "(failed to create, make sure the work folder has correct permissions): " +
+ cacheWorkDir.getAbsolutePath());
+ }
+ else
+ dirExisted = true;
+ }
+
+ return dirExisted;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void sync(int cacheId, int partId) throws IgniteCheckedException {
+ getStore(cacheId, partId).sync();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void ensure(int cacheId, int partId) throws IgniteCheckedException {
+ getStore(cacheId, partId).ensure();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
+ assert partId <= PageIdAllocator.MAX_PARTITION_ID || partId == PageIdAllocator.INDEX_PARTITION;
+
+ PageStore store = getStore(cacheId, partId);
+
+ long pageIdx = store.allocatePage();
+
+ return PageIdUtils.pageId(partId, flags, (int)pageIdx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long metaPageId(final int cacheId) {
+ return metaPageId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int pages(int cacheId, int partId) throws IgniteCheckedException {
+ PageStore store = getStore(cacheId, partId);
+
+ return store.pages();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, StoredCacheData> readCacheConfigurations() throws IgniteCheckedException {
+ if (cctx.kernalContext().clientNode())
+ return Collections.emptyMap();
+
+ File[] files = storeWorkDir.listFiles();
+
+ if (files == null)
+ return Collections.emptyMap();
+
+ Map<String, StoredCacheData> ccfgs = new HashMap<>();
+
+ for (File file : files) {
+ if (file.isDirectory()) {
+ if (file.getName().startsWith(CACHE_DIR_PREFIX)) {
+ File conf = new File(file, CACHE_DATA_FILENAME);
+
+ if (conf.exists() && conf.length() > 0) {
+ StoredCacheData cacheData = readCacheData(conf);
+
+ ccfgs.put(cacheData.config().getName(), cacheData);
+ }
+ }
+ else if (file.getName().startsWith(CACHE_GRP_DIR_PREFIX))
+ readCacheGroupCaches(file, ccfgs);
+ }
+ }
+
+ return ccfgs;
+ }
+
+ /**
+ * @param grpDir Group directory.
+ * @param ccfgs Cache configurations.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void readCacheGroupCaches(File grpDir, Map<String, StoredCacheData> ccfgs) throws IgniteCheckedException {
+ File[] files = grpDir.listFiles();
+
+ if (files == null)
+ return;
+
+ for (File file : files) {
+ if (!file.isDirectory() && file.getName().endsWith(CACHE_DATA_FILENAME) && file.length() > 0) {
+ StoredCacheData cacheData = readCacheData(file);
+
+ ccfgs.put(cacheData.config().getName(), cacheData);
+ }
+ }
+ }
+
+ /**
+ * @param conf File with stored cache data.
+ * @return Cache data.
+ * @throws IgniteCheckedException If failed.
+ */
+ private StoredCacheData readCacheData(File conf) throws IgniteCheckedException {
+ try (InputStream stream = new BufferedInputStream(new FileInputStream(conf))) {
+ return marshaller.unmarshal(stream, U.resolveClassLoader(igniteCfg));
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException("Failed to read cache configuration from disk for cache: " +
+ conf.getAbsolutePath(), e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean hasIndexStore(int grpId) {
+ return !grpsWithoutIdx.contains(grpId);
+ }
+
+ /**
+ * @return Store work dir.
+ */
+ public File workDir() {
+ return storeWorkDir;
+ }
+
+ /**
+ * @param ccfg Cache configuration.
+ * @return Store dir for given cache.
+ */
+ public File cacheWorkDir(CacheConfiguration ccfg) {
+ String dirName = ccfg.getGroupName() == null ?
+ CACHE_DIR_PREFIX + ccfg.getName() : CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
+
+ return new File(storeWorkDir, dirName);
+ }
+
+ /**
+ * @param cleanFiles {@code True} if the stores should delete it's files upon close.
+ */
+ private IgniteCheckedException shutdown(boolean cleanFiles) {
+ IgniteCheckedException ex = null;
+
+ for (CacheStoreHolder holder : idxCacheStores.values())
+ ex = shutdown(holder, cleanFiles, ex);
+
+ return ex;
+ }
+
+ /**
+ * @param holder Store holder.
+ * @param cleanFile {@code True} if files should be cleaned.
+ * @param aggr Aggregating exception.
+ * @return Aggregating exception, if error occurred.
+ */
+ private IgniteCheckedException shutdown(CacheStoreHolder holder, boolean cleanFile,
+ @Nullable IgniteCheckedException aggr) {
+ aggr = shutdown(holder.idxStore, cleanFile, aggr);
+
+ for (FilePageStore store : holder.partStores) {
+ if (store != null)
+ aggr = shutdown(store, cleanFile, aggr);
+ }
+
+ return aggr;
+ }
+
+ /**
+ * @param store Store to shutdown.
+ * @param cleanFile {@code True} if files should be cleaned.
+ * @param aggr Aggregating exception.
+ * @return Aggregating exception, if error occurred.
+ */
+ private IgniteCheckedException shutdown(FilePageStore store, boolean cleanFile, IgniteCheckedException aggr) {
+ try {
+ if (store != null)
+ store.stop(cleanFile);
+ }
+ catch (IgniteCheckedException e) {
+ if (aggr == null)
+ aggr = new IgniteCheckedException("Failed to gracefully shutdown store");
+
+ aggr.addSuppressed(e);
+ }
+
+ return aggr;
+ }
+
+ /**
+ * @param grpId Cache group ID.
+ * @param partId Partition ID.
+ * @return Page store for the corresponding parameters.
+ * @throws IgniteCheckedException If cache or partition with the given ID was not created.
+ *
+ * Note: visible for testing.
+ */
+ public PageStore getStore(int grpId, int partId) throws IgniteCheckedException {
+ CacheStoreHolder holder = idxCacheStores.get(grpId);
+
+ if (holder == null)
+ throw new IgniteCheckedException("Failed to get page store for the given cache ID " +
+ "(cache has not been started): " + grpId);
+
+ if (partId == PageIdAllocator.INDEX_PARTITION)
+ return holder.idxStore;
+
+ if (partId > PageIdAllocator.MAX_PARTITION_ID)
+ throw new IgniteCheckedException("Partition ID is reserved: " + partId);
+
+ FilePageStore store = holder.partStores[partId];
+
+ if (store == null)
+ throw new IgniteCheckedException("Failed to get page store for the given partition ID " +
+ "(partition has not been created) [grpId=" + grpId + ", partId=" + partId + ']');
+
+ return store;
+ }
+
+ /**
+ *
+ */
+ private static class CacheStoreHolder {
+ /** Index store. */
+ private final FilePageStore idxStore;
+
+ /** Partition stores. */
+ private final FilePageStore[] partStores;
+
+ /**
+ *
+ */
+ public CacheStoreHolder(FilePageStore idxStore, FilePageStore[] partStores) {
+ this.idxStore = idxStore;
+ this.partStores = partStores;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
new file mode 100644
index 0000000..d2f0099
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeList.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+
+/**
+ */
+public interface FreeList {
+ /**
+ * @param row Row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void insertDataRow(CacheDataRow row) throws IgniteCheckedException;
+
+ /**
+ * @param link Row link.
+ * @param row New row data.
+ * @return {@code True} if was able to update row.
+ * @throws IgniteCheckedException If failed.
+ */
+ public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException;
+
+ /**
+ * @param link Row link.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void removeDataRowByLink(long link) throws IgniteCheckedException;
+
+ /**
+ * @param log Logger.
+ */
+ public void dumpStatistics(IgniteLogger log);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
new file mode 100644
index 0000000..4e8b8d8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/FreeListImpl.java
@@ -0,0 +1,594 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.freelist;
+
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
+import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicy;
+import org.apache.ignite.internal.processors.cache.persistence.evict.PageEvictionTracker;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.CacheVersionIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.DataPagePayload;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseBag;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ */
+public class FreeListImpl extends PagesList implements FreeList, ReuseList {
+ /** */
+ private static final int BUCKETS = 256; // Must be power of 2.
+
+ /** */
+ private static final int REUSE_BUCKET = BUCKETS - 1;
+
+ /** */
+ private static final Integer COMPLETE = Integer.MAX_VALUE;
+
+ /** */
+ private static final Integer FAIL_I = Integer.MIN_VALUE;
+
+ /** */
+ private static final Long FAIL_L = Long.MAX_VALUE;
+
+ /** */
+ private static final int MIN_PAGE_FREE_SPACE = 8;
+
+ /** */
+ private final int shift;
+
+ /** */
+ private final AtomicReferenceArray<Stripe[]> buckets = new AtomicReferenceArray<>(BUCKETS);
+
+ /** */
+ private final int MIN_SIZE_FOR_DATA_PAGE;
+
+ /** */
+ private final int emptyDataPagesBucket;
+
+ /** */
+ private final PageHandler<CacheDataRow, Boolean> updateRow = new UpdateRowHandler();
+
+ /** */
+ private final MemoryMetricsImpl memMetrics;
+
+ /** */
+ private final PageEvictionTracker evictionTracker;
+
+ /**
+ *
+ */
+ private final class UpdateRowHandler extends PageHandler<CacheDataRow, Boolean> {
+ @Override public Boolean run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ CacheDataRow row,
+ int itemId)
+ throws IgniteCheckedException {
+ DataPageIO io = (DataPageIO)iox;
+
+ int rowSize = getRowSize(row);
+
+ boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize);
+
+ evictionTracker.touchPage(pageId);
+
+ if (updated && needWalDeltaRecord(pageId, page, walPlc)) {
+ // TODO This record must contain only a reference to a logical WAL record with the actual data.
+ byte[] payload = new byte[rowSize];
+
+ DataPagePayload data = io.readPayload(pageAddr, itemId, pageSize());
+
+ assert data.payloadSize() == rowSize;
+
+ PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
+
+ wal.log(new DataPageUpdateRecord(
+ cacheId,
+ pageId,
+ itemId,
+ payload));
+ }
+
+ return updated;
+ }
+ }
+
+ /** */
+ private final PageHandler<CacheDataRow, Integer> writeRow = new WriteRowHandler();
+
+ /**
+ *
+ */
+ private final class WriteRowHandler extends PageHandler<CacheDataRow, Integer> {
+ @Override public Integer run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ CacheDataRow row,
+ int written)
+ throws IgniteCheckedException {
+ DataPageIO io = (DataPageIO)iox;
+
+ int rowSize = getRowSize(row);
+ int oldFreeSpace = io.getFreeSpace(pageAddr);
+
+ assert oldFreeSpace > 0 : oldFreeSpace;
+
+ // If the full row does not fit into this page write only a fragment.
+ written = (written == 0 && oldFreeSpace >= rowSize) ? addRow(pageId, page, pageAddr, io, row, rowSize):
+ addRowFragment(pageId, page, pageAddr, io, row, written, rowSize);
+
+ // Reread free space after update.
+ int newFreeSpace = io.getFreeSpace(pageAddr);
+
+ if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
+ int bucket = bucket(newFreeSpace, false);
+
+ put(null, pageId, page, pageAddr, bucket);
+ }
+
+ if (written == rowSize)
+ evictionTracker.touchPage(pageId);
+
+ // Avoid boxing with garbage generation for usual case.
+ return written == rowSize ? COMPLETE : written;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param row Row.
+ * @param rowSize Row size.
+ * @return Written size which is always equal to row size here.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int addRow(
+ long pageId,
+ long page,
+ long pageAddr,
+ DataPageIO io,
+ CacheDataRow row,
+ int rowSize
+ ) throws IgniteCheckedException {
+ io.addRow(pageAddr, row, rowSize, pageSize());
+
+ if (needWalDeltaRecord(pageId, page, null)) {
+ // TODO This record must contain only a reference to a logical WAL record with the actual data.
+ byte[] payload = new byte[rowSize];
+
+ DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
+
+ assert data.payloadSize() == rowSize;
+
+ PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
+
+ wal.log(new DataPageInsertRecord(
+ cacheId,
+ pageId,
+ payload));
+ }
+
+ return rowSize;
+ }
+
+ /**
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param row Row.
+ * @param written Written size.
+ * @param rowSize Row size.
+ * @return Updated written size.
+ * @throws IgniteCheckedException If failed.
+ */
+ private int addRowFragment(
+ long pageId,
+ long page,
+ long pageAddr,
+ DataPageIO io,
+ CacheDataRow row,
+ int written,
+ int rowSize
+ ) throws IgniteCheckedException {
+ // Read last link before the fragment write, because it will be updated there.
+ long lastLink = row.link();
+
+ int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
+
+ assert payloadSize > 0 : payloadSize;
+
+ if (needWalDeltaRecord(pageId, page, null)) {
+ // TODO This record must contain only a reference to a logical WAL record with the actual data.
+ byte[] payload = new byte[payloadSize];
+
+ DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
+
+ PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize);
+
+ wal.log(new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink));
+ }
+
+ return written + payloadSize;
+ }
+ }
+
+
+ /** */
+ private final PageHandler<Void, Long> rmvRow = new RemoveRowHandler();
+
+ /**
+ *
+ */
+ private final class RemoveRowHandler extends PageHandler<Void, Long> {
+ @Override public Long run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO iox,
+ Boolean walPlc,
+ Void ignored,
+ int itemId)
+ throws IgniteCheckedException {
+ DataPageIO io = (DataPageIO)iox;
+
+ int oldFreeSpace = io.getFreeSpace(pageAddr);
+
+ assert oldFreeSpace >= 0: oldFreeSpace;
+
+ long nextLink = io.removeRow(pageAddr, itemId, pageSize());
+
+ if (needWalDeltaRecord(pageId, page, walPlc))
+ wal.log(new DataPageRemoveRecord(cacheId, pageId, itemId));
+
+ int newFreeSpace = io.getFreeSpace(pageAddr);
+
+ if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
+ int newBucket = bucket(newFreeSpace, false);
+
+ if (oldFreeSpace > MIN_PAGE_FREE_SPACE) {
+ int oldBucket = bucket(oldFreeSpace, false);
+
+ if (oldBucket != newBucket) {
+ // It is possible that page was concurrently taken for put, in this case put will handle bucket change.
+ if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
+ put(null, pageId, page, pageAddr, newBucket);
+ }
+ }
+ else
+ put(null, pageId, page, pageAddr, newBucket);
+
+ if (io.isEmpty(pageAddr))
+ evictionTracker.forgetPage(pageId);
+ }
+
+ // For common case boxed 0L will be cached inside of Long, so no garbage will be produced.
+ return nextLink;
+ }
+ }
+
+ /**
+ * @param cacheId Cache ID.
+ * @param name Name (for debug purpose).
+ * @param memMetrics Memory metrics.
+ * @param memPlc Memory policy.
+ * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself.
+ * @param wal Write ahead log manager.
+ * @param metaPageId Metadata page ID.
+ * @param initNew {@code True} if new metadata should be initialized.
+ * @throws IgniteCheckedException If failed.
+ */
+ public FreeListImpl(
+ int cacheId,
+ String name,
+ MemoryMetricsImpl memMetrics,
+ MemoryPolicy memPlc,
+ ReuseList reuseList,
+ IgniteWriteAheadLogManager wal,
+ long metaPageId,
+ boolean initNew) throws IgniteCheckedException {
+ super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
+ this.evictionTracker = memPlc.evictionTracker();
+ this.reuseList = reuseList == null ? this : reuseList;
+ int pageSize = pageMem.pageSize();
+
+ assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize;
+ assert U.isPow2(BUCKETS);
+ assert BUCKETS <= pageSize : pageSize;
+
+ // TODO this constant is used because currently we cannot reuse data pages as index pages
+ // TODO and vice-versa. It should be removed when data storage format is finalized.
+ MIN_SIZE_FOR_DATA_PAGE = pageSize - DataPageIO.MIN_DATA_PAGE_OVERHEAD;
+
+ int shift = 0;
+
+ while (pageSize > BUCKETS) {
+ shift++;
+ pageSize >>>= 1;
+ }
+
+ this.shift = shift;
+
+ this.memMetrics = memMetrics;
+
+ emptyDataPagesBucket = bucket(MIN_SIZE_FOR_DATA_PAGE, false);
+
+ init(metaPageId, initNew);
+ }
+
+ /**
+ * Calculates average fill factor over FreeListImpl instance.
+ */
+ public float fillFactor() {
+ long pageSize = pageSize();
+
+ long totalSize = 0;
+ long loadSize = 0;
+
+ for (int b = BUCKETS - 2; b > 0; b--) {
+ long bsize = pageSize - ((REUSE_BUCKET - b) << shift);
+
+ long pages = bucketsSize[b].longValue();
+
+ loadSize += pages * (pageSize - bsize);
+
+ totalSize += pages * pageSize;
+ }
+
+ return totalSize == 0 ? -1L : ((float) loadSize / totalSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void dumpStatistics(IgniteLogger log) {
+ long dataPages = 0;
+
+ final boolean dumpBucketsInfo = false;
+
+ for (int b = 0; b < BUCKETS; b++) {
+ long size = bucketsSize[b].longValue();
+
+ if (!isReuseBucket(b))
+ dataPages += size;
+
+ if (dumpBucketsInfo) {
+ Stripe[] stripes = getBucket(b);
+
+ boolean empty = true;
+
+ if (stripes != null) {
+ for (Stripe stripe : stripes) {
+ if (!stripe.empty) {
+ empty = false;
+
+ break;
+ }
+ }
+ }
+
+ log.info("Bucket [b=" + b +
+ ", size=" + size +
+ ", stripes=" + (stripes != null ? stripes.length : 0) +
+ ", stripesEmpty=" + empty + ']');
+ }
+ }
+
+ if (dataPages > 0) {
+ log.info("FreeList [name=" + name +
+ ", buckets=" + BUCKETS +
+ ", dataPages=" + dataPages +
+ ", reusePages=" + bucketsSize[REUSE_BUCKET].longValue() + "]");
+ }
+ }
+
+ /**
+ * @param freeSpace Page free space.
+ * @param allowReuse {@code True} if it is allowed to get reuse bucket.
+ * @return Bucket.
+ */
+ private int bucket(int freeSpace, boolean allowReuse) {
+ assert freeSpace > 0 : freeSpace;
+
+ int bucket = freeSpace >>> shift;
+
+ assert bucket >= 0 && bucket < BUCKETS : bucket;
+
+ if (!allowReuse && isReuseBucket(bucket))
+ bucket--;
+
+ return bucket;
+ }
+
+ /**
+ * @param part Partition.
+ * @return Page ID.
+ * @throws IgniteCheckedException If failed.
+ */
+ private long allocateDataPage(int part) throws IgniteCheckedException {
+ assert part <= PageIdAllocator.MAX_PARTITION_ID;
+ assert part != PageIdAllocator.INDEX_PARTITION;
+
+ return pageMem.allocatePage(cacheId, part, PageIdAllocator.FLAG_DATA);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException {
+ int rowSize = getRowSize(row);
+
+ int written = 0;
+
+ do {
+ if (written != 0)
+ memMetrics.incrementLargeEntriesPages();
+
+ int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written);
+
+ long pageId = 0L;
+
+ if (freeSpace == MIN_SIZE_FOR_DATA_PAGE)
+ pageId = takeEmptyPage(emptyDataPagesBucket, DataPageIO.VERSIONS);
+
+ boolean reuseBucket = false;
+
+ // TODO: properly handle reuse bucket.
+ if (pageId == 0L) {
+ for (int b = bucket(freeSpace, false) + 1; b < BUCKETS - 1; b++) {
+ pageId = takeEmptyPage(b, DataPageIO.VERSIONS);
+
+ if (pageId != 0L) {
+ reuseBucket = isReuseBucket(b);
+
+ break;
+ }
+ }
+ }
+
+ boolean allocated = pageId == 0L;
+
+ if (allocated)
+ pageId = allocateDataPage(row.partition());
+
+ DataPageIO init = reuseBucket || allocated ? DataPageIO.VERSIONS.latest() : null;
+
+ written = write(pageId, writeRow, init, row, written, FAIL_I);
+
+ assert written != FAIL_I; // We can't fail here.
+ }
+ while (written != COMPLETE);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException {
+ assert link != 0;
+
+ long pageId = PageIdUtils.pageId(link);
+ int itemId = PageIdUtils.itemId(link);
+
+ Boolean updated = write(pageId, updateRow, row, itemId, null);
+
+ assert updated != null; // Can't fail here.
+
+ return updated;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void removeDataRowByLink(long link) throws IgniteCheckedException {
+ assert link != 0;
+
+ long pageId = PageIdUtils.pageId(link);
+ int itemId = PageIdUtils.itemId(link);
+
+ long nextLink = write(pageId, rmvRow, itemId, FAIL_L);
+
+ assert nextLink != FAIL_L; // Can't fail here.
+
+ while (nextLink != 0L) {
+ memMetrics.decrementLargeEntriesPages();
+
+ itemId = PageIdUtils.itemId(nextLink);
+ pageId = PageIdUtils.pageId(nextLink);
+
+ nextLink = write(pageId, rmvRow, itemId, FAIL_L);
+
+ assert nextLink != FAIL_L; // Can't fail here.
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Stripe[] getBucket(int bucket) {
+ return buckets.get(bucket);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
+ return buckets.compareAndSet(bucket, exp, upd);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean isReuseBucket(int bucket) {
+ return bucket == REUSE_BUCKET;
+ }
+
+ /**
+ * @return Number of empty data pages in free list.
+ */
+ public int emptyDataPages() {
+ return bucketsSize[emptyDataPagesBucket].intValue();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
+ assert reuseList == this: "not allowed to be a reuse list";
+
+ put(bag, 0, 0, 0L, REUSE_BUCKET);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long takeRecycledPage() throws IgniteCheckedException {
+ assert reuseList == this: "not allowed to be a reuse list";
+
+ return takeEmptyPage(REUSE_BUCKET, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long recycledPagesCount() throws IgniteCheckedException {
+ assert reuseList == this: "not allowed to be a reuse list";
+
+ return storedPagesCount(REUSE_BUCKET);
+ }
+
+ /**
+ * @param row Row.
+ * @return Entry size on page.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static int getRowSize(CacheDataRow row) throws IgniteCheckedException {
+ int keyLen = row.key().valueBytesLength(null);
+ int valLen = row.value().valueBytesLength(null);
+
+ return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (row.cacheId() == 0 ? 0 : 4);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "FreeList [name=" + name + ']';
+ }
+}
[30/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
new file mode 100644
index 0000000..48b0da1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+/**
+ * Registry for IO versions.
+ */
+public final class IOVersions<V extends PageIO> {
+ /** */
+ private final V[] vers;
+
+ /** */
+ private final int type;
+
+ /** */
+ private final V latest;
+
+ /**
+ * @param vers Versions.
+ */
+ @SafeVarargs
+ public IOVersions(V... vers) {
+ assert vers != null;
+ assert vers.length > 0;
+
+ this.vers = vers;
+ this.type = vers[0].getType();
+
+ latest = vers[vers.length - 1];
+
+ assert checkVersions();
+ }
+
+ /**
+ * @return Type.
+ */
+ public int getType() {
+ return type;
+ }
+
+ /**
+ * @return {@code true} If versions are correct.
+ */
+ private boolean checkVersions() {
+ for (int i = 0; i < vers.length; i++) {
+ V v = vers[i];
+
+ if (v.getType() != type || v.getVersion() != i + 1)
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * @return Latest IO version.
+ */
+ public V latest() {
+ return latest;
+ }
+
+ /**
+ * @param ver Version.
+ * @return IO.
+ */
+ public V forVersion(int ver) {
+ return vers[ver - 1];
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return IO.
+ */
+ public V forPage(long pageAddr) {
+ int ver = PageIO.getVersion(pageAddr);
+
+ V res = forVersion(ver);
+
+ assert res.getType() == PageIO.getType(pageAddr) : "resType=" + res.getType() +
+ ", pageType=" + PageIO.getType(pageAddr);
+
+ return res;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
new file mode 100644
index 0000000..6fb5fd0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
@@ -0,0 +1,548 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManagerImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MetadataStorage;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.io.PagesListNodeIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageLockListener;
+
+/**
+ * Base format for all the page types.
+ *
+ * Checklist for page IO implementations and usage (The Rules):
+ *
+ * 1. IO should not have any `public static` methods.
+ * We have versioned IOs and any static method will mean that it have to always work in backward
+ * compatible way between all the IO versions. The base class {@link PageIO} has
+ * static methods (like {@code {@link #getPageId(long)}}) intentionally:
+ * this base format can not be changed between versions.
+ *
+ * 2. IO must correctly override {@link #initNewPage(long, long, int)} method and call super.
+ * We have logic that relies on this behavior.
+ *
+ * 3. Page IO type ID constant must be declared in this class to have a list of all the
+ * existing IO types in a single place.
+ *
+ * 4. IO must be added to {@link #getBPlusIO(int, int)} or to {@link #getPageIO(int, int)}.
+ *
+ * 5. Always keep in mind that IOs are versioned and their format can change from version
+ * to version. In this respect it is a good practice to avoid exposing details
+ * of IO internal format on it's API. The API should be minimalistic and abstract, so that
+ * internal format in future IO version can be completely changed without any changes
+ * to the API of this page IO.
+ *
+ * 6. Page IO API should not have any version dependent semantics and should not change API
+ * semantics in newer versions.
+ *
+ * 7. It is almost always preferable to read or write (especially write) page contents using
+ * static methods on {@link PageHandler}. To just initialize new page use
+ * {@link PageHandler#initPage(PageMemory, int, long, PageIO, IgniteWriteAheadLogManager, PageLockListener)}
+ * method with needed IO instance.
+ */
+public abstract class PageIO {
+ /** */
+ private static BPlusInnerIO<?> innerTestIO;
+
+ /** */
+ private static BPlusLeafIO<?> leafTestIO;
+
+ /** */
+ private static IOVersions<? extends BPlusInnerIO<?>> h2InnerIOs;
+
+ /** */
+ private static IOVersions<? extends BPlusLeafIO<?>> h2LeafIOs;
+
+ /** Maximum payload size. */
+ public static final short MAX_PAYLOAD_SIZE = 2048;
+
+ /** */
+ private static List<IOVersions<? extends BPlusInnerIO<?>>> h2ExtraInnerIOs = new ArrayList<>(MAX_PAYLOAD_SIZE);
+
+ /** */
+ private static List<IOVersions<? extends BPlusLeafIO<?>>> h2ExtraLeafIOs = new ArrayList<>(MAX_PAYLOAD_SIZE);
+
+ /** */
+ public static final int TYPE_OFF = 0;
+
+ /** */
+ public static final int VER_OFF = TYPE_OFF + 2;
+
+ /** */
+ public static final int CRC_OFF = VER_OFF + 2;
+
+ /** */
+ public static final int PAGE_ID_OFF = CRC_OFF + 4;
+
+ /** */
+ private static final int RESERVED_1_OFF = PAGE_ID_OFF + 8;
+
+ /** */
+ private static final int RESERVED_2_OFF = RESERVED_1_OFF + 8;
+
+ /** */
+ private static final int RESERVED_3_OFF = RESERVED_2_OFF + 8;
+
+ /** */
+ public static final int COMMON_HEADER_END = RESERVED_3_OFF + 8; // 40=type(2)+ver(2)+crc(4)+pageId(8)+reserved(3*8)
+
+ /* All the page types. */
+
+ /** */
+ public static final short T_DATA = 1;
+
+ /** */
+ public static final short T_BPLUS_META = 2;
+
+ /** */
+ public static final short T_H2_REF_LEAF = 3;
+
+ /** */
+ public static final short T_H2_REF_INNER = 4;
+
+ /** */
+ public static final short T_DATA_REF_INNER = 5;
+
+ /** */
+ public static final short T_DATA_REF_LEAF = 6;
+
+ /** */
+ public static final short T_METASTORE_INNER = 7;
+
+ /** */
+ public static final short T_METASTORE_LEAF = 8;
+
+ /** */
+ public static final short T_PENDING_REF_INNER = 9;
+
+ /** */
+ public static final short T_PENDING_REF_LEAF = 10;
+
+ /** */
+ public static final short T_META = 11;
+
+ /** */
+ public static final short T_PAGE_LIST_META = 12;
+
+ /** */
+ public static final short T_PAGE_LIST_NODE = 13;
+
+ /** */
+ public static final short T_PART_META = 14;
+
+ /** */
+ public static final short T_PAGE_UPDATE_TRACKING = 15;
+
+ /** */
+ public static final short T_CACHE_ID_AWARE_DATA_REF_INNER = 16;
+
+ /** */
+ public static final short T_CACHE_ID_AWARE_DATA_REF_LEAF = 17;
+
+ /** */
+ public static final short T_CACHE_ID_AWARE_PENDING_REF_INNER = 18;
+
+ /** */
+ public static final short T_CACHE_ID_AWARE_PENDING_REF_LEAF = 19;
+
+ /** */
+ public static final short T_PART_CNTRS = 20;
+
+ /** Index for payload == 1. */
+ public static final short T_H2_EX_REF_LEAF_START = 10000;
+
+ /** */
+ public static final short T_H2_EX_REF_LEAF_END = T_H2_EX_REF_LEAF_START + MAX_PAYLOAD_SIZE - 1;
+
+ /** */
+ public static final short T_H2_EX_REF_INNER_START = 20000;
+
+ /** */
+ public static final short T_H2_EX_REF_INNER_END = T_H2_EX_REF_INNER_START + MAX_PAYLOAD_SIZE - 1;
+
+ /** */
+ private final int ver;
+
+ /** */
+ private final int type;
+
+ /**
+ * @param type Page type.
+ * @param ver Page format version.
+ */
+ protected PageIO(int type, int ver) {
+ assert ver > 0 && ver < 65535: ver;
+ assert type > 0 && type < 65535: type;
+
+ this.type = type;
+ this.ver = ver;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @return Page type.
+ */
+ public static int getType(ByteBuffer buf) {
+ return buf.getShort(TYPE_OFF) & 0xFFFF;
+ }
+
+ /**
+ * @param pageAddr Page addres.
+ * @return Page type.
+ */
+ public static int getType(long pageAddr) {
+ return PageUtils.getShort(pageAddr, TYPE_OFF) & 0xFFFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param type Type.
+ */
+ public static void setType(long pageAddr, int type) {
+ PageUtils.putShort(pageAddr, TYPE_OFF, (short)type);
+
+ assert getType(pageAddr) == type;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @return Version.
+ */
+ public static int getVersion(ByteBuffer buf) {
+ return buf.getShort(VER_OFF) & 0xFFFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Version.
+ */
+ public static int getVersion(long pageAddr) {
+ return PageUtils.getShort(pageAddr, VER_OFF) & 0xFFFF;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param ver Version.
+ */
+ private static void setVersion(long pageAddr, int ver) {
+ PageUtils.putShort(pageAddr, VER_OFF, (short)ver);
+
+ assert getVersion(pageAddr) == ver;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @return Page ID.
+ */
+ public static long getPageId(ByteBuffer buf) {
+ return buf.getLong(PAGE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Page ID.
+ */
+ public static long getPageId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, PAGE_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageId Page ID.
+ */
+ public static void setPageId(long pageAddr, long pageId) {
+ PageUtils.putLong(pageAddr, PAGE_ID_OFF, pageId);
+
+ assert getPageId(pageAddr) == pageId;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Checksum.
+ */
+ public static int getCrc(long pageAddr) {
+ return PageUtils.getInt(pageAddr, CRC_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param crc Checksum.
+ */
+ public static void setCrc(long pageAddr, int crc) {
+ PageUtils.putInt(pageAddr, CRC_OFF, crc);
+ }
+
+ /**
+ * @param buf Buffer.
+ * @return Checksum.
+ */
+ public static int getCrc(ByteBuffer buf) {
+ return buf.getInt(CRC_OFF);
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param crc Checksum.
+ */
+ public static void setCrc(ByteBuffer buf, int crc) {
+ buf.putInt(CRC_OFF, crc);
+ }
+
+ /**
+ * Registers this B+Tree IO versions.
+ *
+ * @param innerIOs Inner IO versions.
+ * @param leafIOs Leaf IO versions.
+ */
+ public static void registerH2(
+ IOVersions<? extends BPlusInnerIO<?>> innerIOs,
+ IOVersions<? extends BPlusLeafIO<?>> leafIOs
+ ) {
+ h2InnerIOs = innerIOs;
+ h2LeafIOs = leafIOs;
+ }
+
+ /**
+ * Registers extra inner IO versions.
+ *
+ * @param innerExtIOs Extra versions.
+ */
+ public static void registerH2ExtraInner(IOVersions<? extends BPlusInnerIO<?>> innerExtIOs) {
+ h2ExtraInnerIOs.add(innerExtIOs);
+ }
+
+ /**
+ * Registers extra inner IO versions.
+ *
+ * @param leafExtIOs Extra versions.
+ */
+ public static void registerH2ExtraLeaf(IOVersions<? extends BPlusLeafIO<?>> leafExtIOs) {
+ h2ExtraLeafIOs.add(leafExtIOs);
+ }
+
+ /**
+ * @param idx Index.
+ * @return IOVersions for given idx.
+ */
+ public static IOVersions<? extends BPlusInnerIO<?>> getInnerVersions(int idx) {
+ return h2ExtraInnerIOs.get(idx);
+ }
+
+ /**
+ * @param idx Index.
+ * @return IOVersions for given idx.
+ */
+ public static IOVersions<? extends BPlusLeafIO<?>> getLeafVersions(int idx) {
+ return h2ExtraLeafIOs.get(idx);
+ }
+
+ /**
+ * Registers IOs for testing.
+ *
+ * @param innerIO Inner IO.
+ * @param leafIO Leaf IO.
+ */
+ public static void registerTest(BPlusInnerIO<?> innerIO, BPlusLeafIO<?> leafIO) {
+ innerTestIO = innerIO;
+ leafTestIO = leafIO;
+ }
+
+ /**
+ * @return Type.
+ */
+ public final int getType() {
+ return type;
+ }
+
+ /**
+ * @return Version.
+ */
+ public final int getVersion() {
+ return ver;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageId Page ID.
+ * @param pageSize Page size.
+ */
+ public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ setType(pageAddr, getType());
+ setVersion(pageAddr, getVersion());
+ setPageId(pageAddr, pageId);
+ setCrc(pageAddr, 0);
+
+ PageUtils.putLong(pageAddr, RESERVED_1_OFF, 0L);
+ PageUtils.putLong(pageAddr, RESERVED_2_OFF, 0L);
+ PageUtils.putLong(pageAddr, RESERVED_3_OFF, 0L);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return getClass().getSimpleName() + "[ver=" + getVersion() + "]";
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return IO.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static <Q extends PageIO> Q getPageIO(long pageAddr) throws IgniteCheckedException {
+ int type = getType(pageAddr);
+ int ver = getVersion(pageAddr);
+
+ return getPageIO(type, ver);
+ }
+
+ /**
+ * @param type IO Type.
+ * @param ver IO Version.
+ * @return Page IO.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public static <Q extends PageIO> Q getPageIO(int type, int ver) throws IgniteCheckedException {
+ switch (type) {
+ case T_DATA:
+ return (Q)DataPageIO.VERSIONS.forVersion(ver);
+
+ case T_BPLUS_META:
+ return (Q)BPlusMetaIO.VERSIONS.forVersion(ver);
+
+ case T_PAGE_LIST_NODE:
+ return (Q)PagesListNodeIO.VERSIONS.forVersion(ver);
+
+ case T_PAGE_LIST_META:
+ return (Q)PagesListMetaIO.VERSIONS.forVersion(ver);
+
+ case T_META:
+ return (Q)PageMetaIO.VERSIONS.forVersion(ver);
+
+ case T_PART_META:
+ return (Q)PagePartitionMetaIO.VERSIONS.forVersion(ver);
+
+ case T_PART_CNTRS:
+ return (Q)PagePartitionCountersIO.VERSIONS.forVersion(ver);
+
+ case T_PAGE_UPDATE_TRACKING:
+ return (Q)TrackingPageIO.VERSIONS.forVersion(ver);
+
+ default:
+ return (Q)getBPlusIO(type, ver);
+ }
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return IO for either inner or leaf B+Tree page.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static <Q extends BPlusIO<?>> Q getBPlusIO(long pageAddr) throws IgniteCheckedException {
+ int type = getType(pageAddr);
+ int ver = getVersion(pageAddr);
+
+ return getBPlusIO(type, ver);
+ }
+
+ /**
+ * @param type IO Type.
+ * @param ver IO Version.
+ * @return IO for either inner or leaf B+Tree page.
+ * @throws IgniteCheckedException If failed.
+ */
+ @SuppressWarnings("unchecked")
+ public static <Q extends BPlusIO<?>> Q getBPlusIO(int type, int ver) throws IgniteCheckedException {
+
+ if (type >= T_H2_EX_REF_LEAF_START && type <= T_H2_EX_REF_LEAF_END)
+ return (Q)h2ExtraLeafIOs.get(type - T_H2_EX_REF_LEAF_START).forVersion(ver);
+
+ if (type >= T_H2_EX_REF_INNER_START && type <= T_H2_EX_REF_INNER_END)
+ return (Q)h2ExtraInnerIOs.get(type - T_H2_EX_REF_INNER_START).forVersion(ver);
+
+ switch (type) {
+ case T_H2_REF_INNER:
+ if (h2InnerIOs == null)
+ break;
+
+ return (Q)h2InnerIOs.forVersion(ver);
+
+ case T_H2_REF_LEAF:
+ if (h2LeafIOs == null)
+ break;
+
+ return (Q)h2LeafIOs.forVersion(ver);
+
+ case T_DATA_REF_INNER:
+ return (Q)IgniteCacheOffheapManagerImpl.DataInnerIO.VERSIONS.forVersion(ver);
+
+ case T_DATA_REF_LEAF:
+ return (Q)IgniteCacheOffheapManagerImpl.DataLeafIO.VERSIONS.forVersion(ver);
+
+ case T_CACHE_ID_AWARE_DATA_REF_INNER:
+ return (Q)IgniteCacheOffheapManagerImpl.CacheIdAwareDataInnerIO.VERSIONS.forVersion(ver);
+
+ case T_CACHE_ID_AWARE_DATA_REF_LEAF:
+ return (Q)IgniteCacheOffheapManagerImpl.CacheIdAwareDataLeafIO.VERSIONS.forVersion(ver);
+
+ case T_METASTORE_INNER:
+ return (Q)MetadataStorage.MetaStoreInnerIO.VERSIONS.forVersion(ver);
+
+ case T_METASTORE_LEAF:
+ return (Q)MetadataStorage.MetaStoreLeafIO.VERSIONS.forVersion(ver);
+
+ case T_PENDING_REF_INNER:
+ return (Q) IgniteCacheOffheapManagerImpl.PendingEntryInnerIO.VERSIONS.forVersion(ver);
+
+ case T_PENDING_REF_LEAF:
+ return (Q)IgniteCacheOffheapManagerImpl.PendingEntryLeafIO.VERSIONS.forVersion(ver);
+
+ case T_CACHE_ID_AWARE_PENDING_REF_INNER:
+ return (Q) IgniteCacheOffheapManagerImpl.CacheIdAwarePendingEntryInnerIO.VERSIONS.forVersion(ver);
+
+ case T_CACHE_ID_AWARE_PENDING_REF_LEAF:
+ return (Q)IgniteCacheOffheapManagerImpl.CacheIdAwarePendingEntryLeafIO.VERSIONS.forVersion(ver);
+
+ default:
+ // For tests.
+ if (innerTestIO != null && innerTestIO.getType() == type && innerTestIO.getVersion() == ver)
+ return (Q)innerTestIO;
+
+ if (leafTestIO != null && leafTestIO.getType() == type && leafTestIO.getVersion() == ver)
+ return (Q)leafTestIO;
+ }
+
+ throw new IgniteCheckedException("Unknown page IO type: " + type);
+ }
+
+ /**
+ * @param type Type to test.
+ * @return {@code True} if data page.
+ */
+ public static boolean isDataPageType(int type) {
+ return type == T_DATA;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
new file mode 100644
index 0000000..ac482e8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageMetaIO.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.pagemem.PageUtils;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ *
+ */
+public class PageMetaIO extends PageIO {
+ /** */
+ private static final int TREE_ROOT_OFF = PageIO.COMMON_HEADER_END + 8;
+
+ /** */
+ private static final int REUSE_LIST_ROOT_OFF = TREE_ROOT_OFF + 8;
+
+ /** Last successful snapshot id offset. */
+ private static final int LAST_SUCCESSFUL_SNAPSHOT_ID_OFF = REUSE_LIST_ROOT_OFF + 8;
+
+ /** Last successful full snapshot id offset. */
+ private static final int LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF = LAST_SUCCESSFUL_SNAPSHOT_ID_OFF + 8;
+
+ /** Next snapshot id offset. */
+ private static final int NEXT_SNAPSHOT_TAG_OFF = LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF + 8;
+
+ /** Last successful full snapshot tag offset. */
+ private static final int LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF = NEXT_SNAPSHOT_TAG_OFF + 8;
+
+ /** Last allocated index offset. */
+ private static final int LAST_PAGE_COUNT_OFF = LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF + 8;
+
+ /** Candidate allocated index offset. */
+ private static final int CANDIDATE_PAGE_COUNT_OFF = LAST_PAGE_COUNT_OFF + 4;
+
+ /** End of page meta. */
+ static final int END_OF_PAGE_META = CANDIDATE_PAGE_COUNT_OFF + 4;
+
+ /** */
+ public static final IOVersions<PageMetaIO> VERSIONS = new IOVersions<>(
+ new PageMetaIO(1)
+ );
+
+ /**
+ * @param ver Page format version.
+ */
+ public PageMetaIO(int ver) {
+ super(PageIO.T_META, ver);
+ }
+
+ /**
+ * @param type Type.
+ * @param ver Version.
+ */
+ protected PageMetaIO(int type, int ver) {
+ super(type, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setTreeRoot(pageAddr, 0);
+ setReuseListRoot(pageAddr, 0);
+ setLastSuccessfulFullSnapshotId(pageAddr, 0);
+ setLastSuccessfulSnapshotId(pageAddr, 0);
+ setNextSnapshotTag(pageAddr, 1);
+ setLastSuccessfulSnapshotTag(pageAddr, 0);
+ setLastPageCount(pageAddr, 0);
+ setCandidatePageCount(pageAddr, 0);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Tree root page.
+ */
+ public long getTreeRoot(long pageAddr) {
+ return PageUtils.getLong(pageAddr, TREE_ROOT_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param treeRoot Tree root
+ */
+ public void setTreeRoot(long pageAddr, long treeRoot) {
+ PageUtils.putLong(pageAddr, TREE_ROOT_OFF, treeRoot);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Reuse list root page.
+ */
+ public long getReuseListRoot(long pageAddr) {
+ return PageUtils.getLong(pageAddr, REUSE_LIST_ROOT_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageId Root page ID.
+ */
+ public void setReuseListRoot(long pageAddr, long pageId) {
+ PageUtils.putLong(pageAddr, REUSE_LIST_ROOT_OFF, pageId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param lastSuccessfulSnapshotId Last successful snapshot id.
+ */
+ public void setLastSuccessfulSnapshotId(long pageAddr, long lastSuccessfulSnapshotId) {
+ PageUtils.putLong(pageAddr, LAST_SUCCESSFUL_SNAPSHOT_ID_OFF, lastSuccessfulSnapshotId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public long getLastSuccessfulSnapshotId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, LAST_SUCCESSFUL_SNAPSHOT_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param lastSuccessfulFullSnapshotId Last successful full snapshot id.
+ */
+ public void setLastSuccessfulFullSnapshotId(long pageAddr, long lastSuccessfulFullSnapshotId) {
+ PageUtils.putLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF, lastSuccessfulFullSnapshotId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public long getLastSuccessfulFullSnapshotId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param nextSnapshotId Next snapshot id.
+ */
+ public void setNextSnapshotTag(long pageAddr, long nextSnapshotId) {
+ PageUtils.putLong(pageAddr, NEXT_SNAPSHOT_TAG_OFF, nextSnapshotId);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public long getLastSuccessfulSnapshotTag(long pageAddr) {
+ return PageUtils.getLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param lastSuccessfulSnapshotTag Last successful snapshot tag.
+ */
+ public void setLastSuccessfulSnapshotTag(long pageAddr, long lastSuccessfulSnapshotTag) {
+ PageUtils.putLong(pageAddr, LAST_SUCCESSFUL_FULL_SNAPSHOT_TAG_OFF, lastSuccessfulSnapshotTag);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public long getNextSnapshotTag(long pageAddr) {
+ return PageUtils.getLong(pageAddr, NEXT_SNAPSHOT_TAG_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageCnt Last allocated index.
+ */
+ public void setLastPageCount(long pageAddr, int pageCnt) {
+ PageUtils.putInt(pageAddr, LAST_PAGE_COUNT_OFF, pageCnt);
+ }
+
+ /**
+ * @param buf Buffer.
+ */
+ public int getLastPageCount(@NotNull ByteBuffer buf) {
+ return buf.getInt(LAST_PAGE_COUNT_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public int getLastPageCount(long pageAddr) {
+ return PageUtils.getInt(pageAddr, LAST_PAGE_COUNT_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param pageCnt Last page count.
+ */
+ public boolean setCandidatePageCount(long pageAddr, int pageCnt) {
+ if (getCandidatePageCount(pageAddr) == pageCnt)
+ return false;
+
+ PageUtils.putInt(pageAddr, CANDIDATE_PAGE_COUNT_OFF, pageCnt);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public int getCandidatePageCount(long pageAddr) {
+ return PageUtils.getInt(pageAddr, CANDIDATE_PAGE_COUNT_OFF);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
new file mode 100644
index 0000000..9bd806f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionCountersIO.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.util.Map;
+import org.apache.ignite.internal.pagemem.PageUtils;
+
+/**
+ *
+ */
+public class PagePartitionCountersIO extends PageIO {
+ /** */
+ private static final int CNT_OFF = COMMON_HEADER_END;
+
+ /** */
+ private static final int LAST_FLAG_OFF = CNT_OFF + 2;
+
+ /** */
+ private static final int NEXT_COUNTERS_PAGE_OFF = LAST_FLAG_OFF + 1;
+
+ /** */
+ private static final int ITEMS_OFF = NEXT_COUNTERS_PAGE_OFF + 8;
+
+ /** */
+ private static final int ITEM_SIZE = 12;
+
+ /** */
+ private static final byte LAST_FLAG = 0b1;
+
+ /** */
+ public static final IOVersions<PagePartitionCountersIO> VERSIONS = new IOVersions<>(
+ new PagePartitionCountersIO(1)
+ );
+
+ /**
+ * @param ver Page format version.
+ */
+ public PagePartitionCountersIO(int ver) {
+ super(T_PART_CNTRS, ver);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setCount(pageAddr, 0);
+ setNextCountersPageId(pageAddr, 0);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Next counters page ID or {@code 0} if it does not exist.
+ */
+ public long getNextCountersPageId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, NEXT_COUNTERS_PAGE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param partMetaPageId Next counters page ID.
+ */
+ public void setNextCountersPageId(long pageAddr, long partMetaPageId) {
+ PageUtils.putLong(pageAddr, NEXT_COUNTERS_PAGE_OFF, partMetaPageId);
+ }
+
+ /**
+ * @param pageSize Page size.
+ * @param pageAddr Page address.
+ * @param cacheSizes Serialized cache size items (pairs of cache ID and its size).
+ * @return Number of written pairs.
+ */
+ public int writeCacheSizes(int pageSize, long pageAddr, byte[] cacheSizes, int itemsOff) {
+ assert cacheSizes != null;
+ assert cacheSizes.length % ITEM_SIZE == 0 : cacheSizes.length;
+
+ int cap = getCapacity(pageSize);
+
+ int items = (cacheSizes.length / ITEM_SIZE) - itemsOff;
+ int write = Math.min(cap, items);
+
+ if (write > 0)
+ // This can happen in case there are no items in a given partition for all caches in the group.
+ PageUtils.putBytes(pageAddr, ITEMS_OFF, cacheSizes, itemsOff * ITEM_SIZE, write * ITEM_SIZE);
+
+ setCount(pageAddr, write);
+
+ setLastFlag(pageAddr, write == items);
+
+ return write;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param res Result map of cache sizes.
+ * @return {@code True} if the map was fully read.
+ */
+ public boolean readCacheSizes(long pageAddr, Map<Integer, Long> res) {
+ int cnt = getCount(pageAddr);
+
+ assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
+
+ if (cnt == 0)
+ return true;
+
+ int off = ITEMS_OFF;
+
+ for (int i = 0; i < cnt; i++) {
+ int cacheId = PageUtils.getInt(pageAddr, off);
+ off += 4;
+
+ assert cacheId != 0;
+
+ long cacheSize = PageUtils.getLong(pageAddr, off);
+ off += 8;
+
+ assert cacheSize > 0 : cacheSize;
+
+ Long old = res.put(cacheId, cacheSize);
+
+ assert old == null;
+ }
+
+ return getLastFlag(pageAddr);
+ }
+
+ private boolean getLastFlag(long pageAddr) {
+ return PageUtils.getByte(pageAddr, LAST_FLAG_OFF) == LAST_FLAG;
+ }
+
+ private void setLastFlag(long pageAddr, boolean last) {
+ PageUtils.putByte(pageAddr, LAST_FLAG_OFF, last ? LAST_FLAG : ~LAST_FLAG);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Stored items count.
+ */
+ private int getCount(long pageAddr) {
+ return PageUtils.getShort(pageAddr, CNT_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cnt Stored items count.
+ */
+ private void setCount(long pageAddr, int cnt) {
+ assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
+
+ PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
+ }
+
+ /**
+ * @param pageSize Page size.
+ * @return Maximum number of items which can be stored in buffer.
+ */
+ private int getCapacity(int pageSize) {
+ return (pageSize - ITEMS_OFF) / ITEM_SIZE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
new file mode 100644
index 0000000..88dbf9c
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PagePartitionMetaIO.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import org.apache.ignite.internal.pagemem.PageUtils;
+
+/**
+ *
+ */
+public class PagePartitionMetaIO extends PageMetaIO {
+ /** */
+ private static final int SIZE_OFF = PageMetaIO.END_OF_PAGE_META;
+
+ /** */
+ private static final int UPDATE_CNTR_OFF = SIZE_OFF + 8;
+
+ /** */
+ private static final int GLOBAL_RMV_ID_OFF = UPDATE_CNTR_OFF + 8;
+
+ /** */
+ private static final int PARTITION_STATE_OFF = GLOBAL_RMV_ID_OFF + 8;
+
+ /** */
+ private static final int NEXT_PART_META_PAGE_OFF = PARTITION_STATE_OFF + 1;
+
+ /** */
+ public static final IOVersions<PagePartitionMetaIO> VERSIONS = new IOVersions<>(
+ new PagePartitionMetaIO(1)
+ );
+
+ /** {@inheritDoc} */
+ @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
+ super.initNewPage(pageAddr, pageId, pageSize);
+
+ setSize(pageAddr, 0);
+ setUpdateCounter(pageAddr, 0);
+ setGlobalRemoveId(pageAddr, 0);
+ setPartitionState(pageAddr, (byte)-1);
+ setCountersPageId(pageAddr, 0);
+ }
+
+ /**
+ * @param ver Version.
+ */
+ public PagePartitionMetaIO(int ver) {
+ super(T_PART_META, ver);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Partition size.
+ */
+ public long getSize(long pageAddr) {
+ return PageUtils.getLong(pageAddr, SIZE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param size Partition size.
+ */
+ public boolean setSize(long pageAddr, long size) {
+ if (getSize(pageAddr) == size)
+ return false;
+
+ PageUtils.putLong(pageAddr, SIZE_OFF, size);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Partition update counter.
+ */
+ public long getUpdateCounter(long pageAddr) {
+ return PageUtils.getLong(pageAddr, UPDATE_CNTR_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param cntr Partition update counter.
+ */
+ public boolean setUpdateCounter(long pageAddr, long cntr) {
+ if (getUpdateCounter(pageAddr) == cntr)
+ return false;
+
+ PageUtils.putLong(pageAddr, UPDATE_CNTR_OFF, cntr);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Global remove ID.
+ */
+ public long getGlobalRemoveId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, GLOBAL_RMV_ID_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param rmvId Global remove ID.
+ */
+ public boolean setGlobalRemoveId(long pageAddr, long rmvId) {
+ if (getGlobalRemoveId(pageAddr) == rmvId)
+ return false;
+
+ PageUtils.putLong(pageAddr, GLOBAL_RMV_ID_OFF, rmvId);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ */
+ public byte getPartitionState(long pageAddr) {
+ return PageUtils.getByte(pageAddr, PARTITION_STATE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address
+ * @param state State.
+ */
+ public boolean setPartitionState(long pageAddr, byte state) {
+ if (getPartitionState(pageAddr) == state)
+ return false;
+
+ PageUtils.putByte(pageAddr, PARTITION_STATE_OFF, state);
+
+ return true;
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @return Next meta partial page ID or {@code 0} if it does not exist.
+ */
+ public long getCountersPageId(long pageAddr) {
+ return PageUtils.getLong(pageAddr, NEXT_PART_META_PAGE_OFF);
+ }
+
+ /**
+ * @param pageAddr Page address.
+ * @param metaPageId Next partial meta page ID.
+ */
+ public void setCountersPageId(long pageAddr, long metaPageId) {
+ PageUtils.putLong(pageAddr, NEXT_PART_META_PAGE_OFF, metaPageId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
new file mode 100644
index 0000000..aa6ba9a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/TrackingPageIO.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.io;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.persistence.tree.util.PageHandler;
+
+/**
+ * We use dedicated page for tracking pages updates.
+ * Also we divide such 'tracking' pages on two half, first is used for check that page was changed or not
+ * (during incremental snapshot), second - to accumulate changed for next snapshot.
+ *
+ * You cannot test change for not started snapshot! because it will cause of preparation for snapshot.
+ *
+ * Implementation. For each page there is own bit in both half. Tracking page is used for tracking N page after it.
+ * N depends on page size (how many bytes we can use for tracking).
+ *
+ *
+ * +-----------------------------------------+-----------------------------------------+
+ * | left half | right half |
+ * +---------+----------+----+------------------------------------+----+------------------------------------+
+ * | HEADER | Last |size| |size| |
+ * | |SnapshotId|2b. | tracking bits |2b. | tracking bits |
+ * +---------+----------+----+------------------------------------+----+------------------------------------+
+ *
+ */
+public class TrackingPageIO extends PageIO {
+ /** */
+ public static final IOVersions<TrackingPageIO> VERSIONS = new IOVersions<>(
+ new TrackingPageIO(1)
+ );
+
+ /** Last snapshot offset. */
+ public static final int LAST_SNAPSHOT_TAG_OFFSET = COMMON_HEADER_END;
+
+ /** Size field offset. */
+ public static final int SIZE_FIELD_OFFSET = LAST_SNAPSHOT_TAG_OFFSET + 8;
+
+ /** 'Size' field size. */
+ public static final int SIZE_FIELD_SIZE = 2;
+
+ /** Bitmap offset. */
+ public static final int BITMAP_OFFSET = SIZE_FIELD_OFFSET + SIZE_FIELD_SIZE;
+
+ /** Count of extra page. */
+ public static final int COUNT_OF_EXTRA_PAGE = 1;
+
+ /**
+ * @param ver Page format version.
+ */
+ protected TrackingPageIO(int ver) {
+ super(PageIO.T_PAGE_UPDATE_TRACKING, ver);
+ }
+
+ /**
+ * Will mark pageId as changed for next (!) snapshotId
+ *
+ * @param buf Buffer.
+ * @param pageId Page id.
+ * @param nextSnapshotTag tag of next snapshot.
+ * @param pageSize Page size.
+ */
+ public boolean markChanged(ByteBuffer buf, long pageId, long nextSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+ validateSnapshotId(buf, nextSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
+
+ int cntOfPage = countOfPageToTrack(pageSize);
+
+ int idxToUpdate = (PageIdUtils.pageIndex(pageId) - COUNT_OF_EXTRA_PAGE) % cntOfPage;
+
+ int sizeOff = useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + (cntOfPage >> 3);
+
+ int idx = sizeOff + SIZE_FIELD_SIZE + (idxToUpdate >> 3);
+
+ byte byteToUpdate = buf.get(idx);
+
+ int updateTemplate = 1 << (idxToUpdate & 0b111);
+
+ byte newVal = (byte) (byteToUpdate | updateTemplate);
+
+ if (byteToUpdate == newVal)
+ return false;
+
+ buf.put(idx, newVal);
+
+ short newSize = (short)(buf.getShort(sizeOff) + 1);
+
+ buf.putShort(sizeOff, newSize);
+
+ assert newSize == countOfChangedPage(buf, nextSnapshotTag, pageSize);
+
+ return true;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param nextSnapshotTag Next snapshot id.
+ * @param lastSuccessfulSnapshotId Last successful snapshot id.
+ * @param pageSize Page size.
+ */
+ private void validateSnapshotId(ByteBuffer buf, long nextSnapshotTag, long lastSuccessfulSnapshotId, int pageSize) {
+ assert nextSnapshotTag != lastSuccessfulSnapshotId : "nextSnapshotTag = " + nextSnapshotTag +
+ ", lastSuccessfulSnapshotId = " + lastSuccessfulSnapshotId;
+
+ long last = getLastSnapshotTag(buf);
+
+ assert last <= nextSnapshotTag : "last = " + last + ", nextSnapshotTag = " + nextSnapshotTag;
+
+ if (nextSnapshotTag == last) //everything is ok
+ return;
+
+ int cntOfPage = countOfPageToTrack(pageSize);
+
+ if (last <= lastSuccessfulSnapshotId) { //we can drop our data
+ buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
+
+ PageHandler.zeroMemory(buf, SIZE_FIELD_OFFSET, buf.capacity() - SIZE_FIELD_OFFSET);
+ } else { //we can't drop data, it is still necessary for incremental snapshots
+ int len = cntOfPage >> 3;
+
+ int sizeOff = useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + len;
+ int sizeOff2 = !useLeftHalf(nextSnapshotTag) ? SIZE_FIELD_OFFSET : BITMAP_OFFSET + len;
+
+ if (last - lastSuccessfulSnapshotId == 1) { //we should keep only data in last half
+ //new data will be written in the same half, we should move old data to another half
+ if ((nextSnapshotTag - last) % 2 == 0)
+ PageHandler.copyMemory(buf, buf, sizeOff, sizeOff2, len + SIZE_FIELD_SIZE);
+ } else { //last - lastSuccessfulSnapshotId > 1, e.g. we should merge two half in one
+ int newSize = 0;
+ int i = 0;
+
+ for (; i < len - 8; i += 8) {
+ long newVal = buf.getLong(sizeOff + SIZE_FIELD_SIZE + i) | buf.getLong(sizeOff2 + SIZE_FIELD_SIZE + i);
+
+ newSize += Long.bitCount(newVal);
+
+ buf.putLong(sizeOff2 + SIZE_FIELD_SIZE + i, newVal);
+ }
+
+ for (; i < len; i ++) {
+ byte newVal = (byte) (buf.get(sizeOff + SIZE_FIELD_SIZE + i) | buf.get(sizeOff2 + SIZE_FIELD_SIZE + i));
+
+ newSize += Integer.bitCount(newVal & 0xFF);
+
+ buf.put(sizeOff2 + SIZE_FIELD_SIZE + i, newVal);
+ }
+
+ buf.putShort(sizeOff2, (short)newSize);
+ }
+
+ buf.putLong(LAST_SNAPSHOT_TAG_OFFSET, nextSnapshotTag);
+
+ PageHandler.zeroMemory(buf, sizeOff, len + SIZE_FIELD_SIZE);
+ }
+ }
+
+ /**
+ * @param buf Buffer.
+ */
+ long getLastSnapshotTag(ByteBuffer buf) {
+ return buf.getLong(LAST_SNAPSHOT_TAG_OFFSET);
+ }
+
+ /**
+ * Check that pageId was marked as changed between previous snapshot finish and current snapshot start.
+ *
+ * @param buf Buffer.
+ * @param pageId Page id.
+ * @param curSnapshotTag Snapshot tag.
+ * @param pageSize Page size.
+ */
+ public boolean wasChanged(ByteBuffer buf, long pageId, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+ validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
+
+ if (countOfChangedPage(buf, curSnapshotTag, pageSize) < 1)
+ return false;
+
+ int cntOfPage = countOfPageToTrack(pageSize);
+
+ int idxToTest = (PageIdUtils.pageIndex(pageId) - COUNT_OF_EXTRA_PAGE) % cntOfPage;
+
+ byte byteToTest;
+
+ if (useLeftHalf(curSnapshotTag))
+ byteToTest = buf.get(BITMAP_OFFSET + (idxToTest >> 3));
+ else
+ byteToTest = buf.get(BITMAP_OFFSET + SIZE_FIELD_SIZE + ((idxToTest + cntOfPage) >> 3));
+
+ int testTemplate = 1 << (idxToTest & 0b111);
+
+ return ((byteToTest & testTemplate) ^ testTemplate) == 0;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param snapshotTag Snapshot tag.
+ * @param pageSize Page size.
+ *
+ * @return count of pages which were marked as change for given snapshotTag
+ */
+ public short countOfChangedPage(ByteBuffer buf, long snapshotTag, int pageSize) {
+ long dif = getLastSnapshotTag(buf) - snapshotTag;
+
+ if (dif != 0 && dif != 1)
+ return -1;
+
+ if (useLeftHalf(snapshotTag))
+ return buf.getShort(SIZE_FIELD_OFFSET);
+ else
+ return buf.getShort(BITMAP_OFFSET + (countOfPageToTrack(pageSize) >> 3));
+ }
+
+ /**
+ * @param snapshotTag Snapshot id.
+ *
+ * @return true if snapshotTag is odd, otherwise - false
+ */
+ boolean useLeftHalf(long snapshotTag) {
+ return (snapshotTag & 0b1) == 0;
+ }
+
+ /**
+ * @param pageId Page id.
+ * @param pageSize Page size.
+ * @return pageId of tracking page which set pageId belongs to
+ */
+ public long trackingPageFor(long pageId, int pageSize) {
+ assert PageIdUtils.pageIndex(pageId) > 0;
+
+ int pageIdx = ((PageIdUtils.pageIndex(pageId) - COUNT_OF_EXTRA_PAGE) /
+ countOfPageToTrack(pageSize)) * countOfPageToTrack(pageSize) + COUNT_OF_EXTRA_PAGE;
+
+ long trackingPageId = PageIdUtils.pageId(PageIdUtils.partId(pageId), PageIdUtils.flag(pageId), pageIdx);
+
+ assert PageIdUtils.pageIndex(trackingPageId) <= PageIdUtils.pageIndex(pageId);
+
+ return trackingPageId;
+ }
+
+ /**
+ * @param pageSize Page size.
+ *
+ * @return how many page we can track with 1 page
+ */
+ public int countOfPageToTrack(int pageSize) {
+ return ((pageSize - SIZE_FIELD_OFFSET) / 2 - SIZE_FIELD_SIZE) << 3;
+ }
+
+ /**
+ * @param buf Buffer.
+ * @param start Start.
+ * @param curSnapshotTag Snapshot id.
+ * @param pageSize Page size.
+ * @return set pageId if it was changed or next closest one, if there is no changed page null will be returned
+ */
+ public Long findNextChangedPage(ByteBuffer buf, long start, long curSnapshotTag, long lastSuccessfulSnapshotTag, int pageSize) {
+ validateSnapshotId(buf, curSnapshotTag + 1, lastSuccessfulSnapshotTag, pageSize);
+
+ int cntOfPage = countOfPageToTrack(pageSize);
+
+ long trackingPage = trackingPageFor(start, pageSize);
+
+ if (start == trackingPage)
+ return trackingPage;
+
+ if (countOfChangedPage(buf, curSnapshotTag, pageSize) <= 0)
+ return null;
+
+ int idxToStartTest = (PageIdUtils.pageIndex(start) - COUNT_OF_EXTRA_PAGE) % cntOfPage;
+
+ int zeroIdx = useLeftHalf(curSnapshotTag)? BITMAP_OFFSET : BITMAP_OFFSET + SIZE_FIELD_SIZE + (cntOfPage >> 3);
+
+ int startIdx = zeroIdx + (idxToStartTest >> 3);
+
+ int idx = startIdx;
+
+ int stopIdx = zeroIdx + (cntOfPage >> 3);
+
+ while (idx < stopIdx) {
+ byte byteToTest = buf.get(idx);
+
+ if (byteToTest != 0) {
+ int foundSetBit;
+ if ((foundSetBit = foundSetBit(byteToTest, idx == startIdx ? (idxToStartTest & 0b111) : 0)) != -1) {
+ long foundPageId = PageIdUtils.pageId(
+ PageIdUtils.partId(start),
+ PageIdUtils.flag(start),
+ PageIdUtils.pageIndex(trackingPage) + ((idx - zeroIdx) << 3) + foundSetBit);
+
+ assert wasChanged(buf, foundPageId, curSnapshotTag, lastSuccessfulSnapshotTag, pageSize);
+ assert trackingPageFor(foundPageId, pageSize) == trackingPage;
+
+ return foundPageId;
+ }
+ }
+
+ idx++;
+ }
+
+ return null;
+ }
+
+ /**
+ * @param byteToTest Byte to test.
+ * @param firstBitToTest First bit to test.
+ */
+ private static int foundSetBit(byte byteToTest, int firstBitToTest) {
+ assert firstBitToTest < 8;
+
+ for (int i = firstBitToTest; i < 8; i++) {
+ int testTemplate = 1 << i;
+
+ if (((byteToTest & testTemplate) ^ testTemplate) == 0)
+ return i;
+ }
+
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseBag.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseBag.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseBag.java
new file mode 100644
index 0000000..843eccf
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseBag.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.reuse;
+
+/**
+ * Reuse bag for free index pages.
+ */
+public interface ReuseBag {
+ /**
+ * @param pageId Free page ID for reuse.
+ */
+ public void addFreePage(long pageId);
+
+ /**
+ * @return Free page ID for reuse or {@code 0} if empty.
+ */
+ public long pollFreePage();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseList.java
new file mode 100644
index 0000000..aaab186
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseList.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.reuse;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * Reuse list.
+ */
+public interface ReuseList {
+ /**
+ * @param bag Reuse bag.
+ * @throws IgniteCheckedException If failed.
+ */
+ public void addForRecycle(ReuseBag bag) throws IgniteCheckedException;
+
+ /**
+ * @return Page ID or {@code 0} if none available.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long takeRecycledPage() throws IgniteCheckedException;
+
+ /**
+ * @return Number of recycled pages it contains.
+ * @throws IgniteCheckedException If failed.
+ */
+ public long recycledPagesCount() throws IgniteCheckedException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseListImpl.java
new file mode 100644
index 0000000..6291c1d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/reuse/ReuseListImpl.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.reuse;
+
+import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
+
+/**
+ * Reuse list.
+ */
+public class ReuseListImpl extends PagesList implements ReuseList {
+ /** */
+ private static final AtomicReferenceFieldUpdater<ReuseListImpl, Stripe[]> bucketUpdater =
+ AtomicReferenceFieldUpdater.newUpdater(ReuseListImpl.class, Stripe[].class, "bucket");
+
+ /** */
+ private volatile Stripe[] bucket;
+
+ /**
+ * @param cacheId Cache ID.
+ * @param name Name (for debug purpose).
+ * @param pageMem Page memory.
+ * @param wal Write ahead log manager.
+ * @param metaPageId Metadata page ID.
+ * @param initNew {@code True} if new metadata should be initialized.
+ * @throws IgniteCheckedException If failed.
+ */
+ public ReuseListImpl(int cacheId,
+ String name,
+ PageMemory pageMem,
+ IgniteWriteAheadLogManager wal,
+ long metaPageId,
+ boolean initNew) throws IgniteCheckedException {
+ super(cacheId, name, pageMem, 1, wal, metaPageId);
+
+ reuseList = this;
+
+ init(metaPageId, initNew);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean isReuseBucket(int bucket) {
+ assert bucket == 0: bucket;
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
+ put(bag, 0, 0, 0, 0);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long takeRecycledPage() throws IgniteCheckedException {
+ return takeEmptyPage(0, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long recycledPagesCount() throws IgniteCheckedException {
+ return storedPagesCount(0);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Stripe[] getBucket(int bucket) {
+ return this.bucket;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
+ return bucketUpdater.compareAndSet(this, exp, upd);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return "ReuseList [name=" + name + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
new file mode 100644
index 0000000..648ddf7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageHandler.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.util;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
+import org.apache.ignite.internal.util.GridUnsafe;
+
+import static java.lang.Boolean.FALSE;
+import static java.lang.Boolean.TRUE;
+
+/**
+ * Page handler.
+ */
+public abstract class PageHandler<X, R> {
+ /** */
+ private static final PageHandler<Void, Boolean> NO_OP = new PageHandler<Void, Boolean>() {
+ @Override public Boolean run(int cacheId, long pageId, long page, long pageAddr, PageIO io, Boolean walPlc,
+ Void arg,
+ int intArg) throws IgniteCheckedException {
+ return Boolean.TRUE;
+ }
+ };
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page absolute pointer.
+ * @param pageAddr Page address.
+ * @param io IO.
+ * @param walPlc Full page WAL record policy.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @return Result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public abstract R run(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO io,
+ Boolean walPlc,
+ X arg,
+ int intArg)
+ throws IgniteCheckedException;
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @return {@code true} If release.
+ */
+ public boolean releaseAfterWrite(
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ X arg,
+ int intArg) {
+ return true;
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param lsnr Lock listener.
+ * @param h Handler.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static <X, R> R readPage(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ PageLockListener lsnr,
+ PageHandler<X, R> h,
+ X arg,
+ int intArg,
+ R lockFailed
+ ) throws IgniteCheckedException {
+ long page = pageMem.acquirePage(cacheId, pageId);
+ try {
+ long pageAddr = readLock(pageMem, cacheId, pageId, page, lsnr);
+
+ if (pageAddr == 0L)
+ return lockFailed;
+ try {
+ PageIO io = PageIO.getPageIO(pageAddr);
+ return h.run(cacheId, pageId, page, pageAddr, io, null, arg, intArg);
+ }
+ finally {
+ readUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr);
+ }
+ }
+ finally {
+ pageMem.releasePage(cacheId, pageId, page);
+ }
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param lsnr Lock listener.
+ * @param h Handler.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static <X, R> R readPage(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ PageLockListener lsnr,
+ PageHandler<X, R> h,
+ X arg,
+ int intArg,
+ R lockFailed
+ ) throws IgniteCheckedException {
+ long pageAddr = readLock(pageMem, cacheId, pageId, page, lsnr);
+
+ if (pageAddr == 0L)
+ return lockFailed;
+ try {
+ PageIO io = PageIO.getPageIO(pageAddr);
+ return h.run(cacheId, pageId, page, pageAddr, io, null, arg, intArg);
+ }
+ finally {
+ readUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr);
+ }
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param lsnr Lock listener.
+ * @return Page address.
+ */
+ public static long readLock(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ PageLockListener lsnr) {
+ lsnr.onBeforeReadLock(cacheId, pageId, page);
+
+ long pageAddr = pageMem.readLock(cacheId, pageId, page);
+
+ lsnr.onReadLock(cacheId, pageId, page, pageAddr);
+
+ return pageAddr;
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address (for-write pointer)
+ * @param lsnr Lock listener.
+ */
+ public static void readUnlock(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageLockListener lsnr) {
+ lsnr.onReadUnlock(cacheId, pageId, page, pageAddr);
+
+ pageMem.readUnlock(cacheId, pageId, page);
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param init IO for new page initialization.
+ * @param wal Write ahead log.
+ * @param lsnr Lock listener.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static void initPage(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ PageIO init,
+ IgniteWriteAheadLogManager wal,
+ PageLockListener lsnr
+ ) throws IgniteCheckedException {
+ Boolean res = writePage(pageMem, cacheId, pageId, lsnr, PageHandler.NO_OP, init, wal, null, null, 0, FALSE);
+
+ assert res != FALSE;
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param lsnr Lock listener.
+ * @param h Handler.
+ * @param init IO for new page initialization or {@code null} if it is an existing page.
+ * @param wal Write ahead log.
+ * @param walPlc Full page WAL record policy.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static <X, R> R writePage(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ PageLockListener lsnr,
+ PageHandler<X, R> h,
+ PageIO init,
+ IgniteWriteAheadLogManager wal,
+ Boolean walPlc,
+ X arg,
+ int intArg,
+ R lockFailed
+ ) throws IgniteCheckedException {
+ boolean releaseAfterWrite = true;
+ long page = pageMem.acquirePage(cacheId, pageId);
+ try {
+ long pageAddr = writeLock(pageMem, cacheId, pageId, page, lsnr, false);
+
+ if (pageAddr == 0L)
+ return lockFailed;
+
+ boolean ok = false;
+
+ try {
+ if (init != null) {
+ // It is a new page and we have to initialize it.
+ doInitPage(pageMem, cacheId, pageId, page, pageAddr, init, wal);
+ walPlc = FALSE;
+ }
+ else {
+ init = PageIO.getPageIO(pageAddr);
+ }
+
+ R res = h.run(cacheId, pageId, page, pageAddr, init, walPlc, arg, intArg);
+
+ ok = true;
+
+ return res;
+ }
+ finally {
+ assert PageIO.getCrc(pageAddr) == 0; //TODO GG-11480
+
+ if (releaseAfterWrite = h.releaseAfterWrite(cacheId, pageId, page, pageAddr, arg, intArg))
+ writeUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr, walPlc, ok);
+ }
+ }
+ finally {
+ if (releaseAfterWrite)
+ pageMem.releasePage(cacheId, pageId, page);
+ }
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param lsnr Lock listener.
+ * @param h Handler.
+ * @param init IO for new page initialization or {@code null} if it is an existing page.
+ * @param wal Write ahead log.
+ * @param walPlc Full page WAL record policy.
+ * @param arg Argument.
+ * @param intArg Argument of type {@code int}.
+ * @param lockFailed Result in case of lock failure due to page recycling.
+ * @return Handler result.
+ * @throws IgniteCheckedException If failed.
+ */
+ public static <X, R> R writePage(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ PageLockListener lsnr,
+ PageHandler<X, R> h,
+ PageIO init,
+ IgniteWriteAheadLogManager wal,
+ Boolean walPlc,
+ X arg,
+ int intArg,
+ R lockFailed
+ ) throws IgniteCheckedException {
+ long pageAddr = writeLock(pageMem, cacheId, pageId, page, lsnr, false);
+
+ if (pageAddr == 0L)
+ return lockFailed;
+
+ boolean ok = false;
+
+ try {
+ if (init != null) {
+ // It is a new page and we have to initialize it.
+ doInitPage(pageMem, cacheId, pageId, page, pageAddr, init, wal);
+ walPlc = FALSE;
+ }
+ else {
+ init = PageIO.getPageIO(pageAddr);
+ }
+
+ R res = h.run(cacheId, pageId, page, pageAddr, init, walPlc, arg, intArg);
+
+ ok = true;
+
+ return res;
+ }
+ finally {
+ assert PageIO.getCrc(pageAddr) == 0; //TODO GG-11480
+
+ if (h.releaseAfterWrite(cacheId, pageId, page, pageAddr, arg, intArg))
+ writeUnlock(pageMem, cacheId, pageId, page, pageAddr, lsnr, walPlc, ok);
+ }
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param lsnr Lock listener.
+ * @param walPlc Full page WAL record policy.
+ * @param dirty Page is dirty.
+ */
+ public static void writeUnlock(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageLockListener lsnr,
+ Boolean walPlc,
+ boolean dirty) {
+ lsnr.onWriteUnlock(cacheId, pageId, page, pageAddr);
+
+ pageMem.writeUnlock(cacheId, pageId, page, walPlc, dirty);
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param lsnr Lock listener.
+ * @param tryLock Only try to lock without waiting.
+ * @return Page address or {@code 0} if failed to lock due to recycling.
+ */
+ public static long writeLock(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ PageLockListener lsnr,
+ boolean tryLock) {
+ lsnr.onBeforeWriteLock(cacheId, pageId, page);
+
+ long pageAddr = tryLock ? pageMem.tryWriteLock(cacheId, pageId, page) : pageMem.writeLock(cacheId, pageId, page);
+
+ lsnr.onWriteLock(cacheId, pageId, page, pageAddr);
+
+ return pageAddr;
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ * @param init Initial IO.
+ * @param wal Write ahead log.
+ * @throws IgniteCheckedException If failed.
+ */
+ private static void doInitPage(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ long pageAddr,
+ PageIO init,
+ IgniteWriteAheadLogManager wal) throws IgniteCheckedException {
+
+ assert PageIO.getCrc(pageAddr) == 0; //TODO GG-11480
+
+ init.initNewPage(pageAddr, pageId, pageMem.pageSize());
+
+ // Here we should never write full page, because it is known to be new.
+ if (isWalDeltaRecordNeeded(pageMem, cacheId, pageId, page, wal, FALSE))
+ wal.log(new InitNewPageRecord(cacheId, pageId,
+ init.getType(), init.getVersion(), pageId));
+ }
+
+ /**
+ * @param pageMem Page memory.
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param wal Write ahead log.
+ * @param walPlc Full page WAL record policy.
+ * @return {@code true} If we need to make a delta WAL record for the change in this page.
+ */
+ public static boolean isWalDeltaRecordNeeded(
+ PageMemory pageMem,
+ int cacheId,
+ long pageId,
+ long page,
+ IgniteWriteAheadLogManager wal,
+ Boolean walPlc) {
+ // If the page is clean, then it is either newly allocated or just after checkpoint.
+ // In both cases we have to write full page contents to WAL.
+ return wal != null && !wal.isAlwaysWriteFullPages() && walPlc != TRUE &&
+ (walPlc == FALSE || pageMem.isDirty(cacheId, pageId, page));
+ }
+
+ /**
+ * @param src Source.
+ * @param dst Destination.
+ * @param srcOff Source offset in bytes.
+ * @param dstOff Destination offset in bytes.
+ * @param cnt Bytes count to copy.
+ */
+ public static void copyMemory(ByteBuffer src, ByteBuffer dst, long srcOff, long dstOff, long cnt) {
+ byte[] srcArr = src.hasArray() ? src.array() : null;
+ byte[] dstArr = dst.hasArray() ? dst.array() : null;
+ long srcArrOff = src.hasArray() ? src.arrayOffset() + GridUnsafe.BYTE_ARR_OFF : 0;
+ long dstArrOff = dst.hasArray() ? dst.arrayOffset() + GridUnsafe.BYTE_ARR_OFF : 0;
+
+ long srcPtr = src.isDirect() ? GridUnsafe.bufferAddress(src) : 0;
+ long dstPtr = dst.isDirect() ? GridUnsafe.bufferAddress(dst) : 0;
+
+ GridUnsafe.copyMemory(srcArr, srcPtr + srcArrOff + srcOff, dstArr, dstPtr + dstArrOff + dstOff, cnt);
+ }
+
+ /**
+ * Will zero memory in buf
+ * @param buf Buffer.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public static void zeroMemory(ByteBuffer buf, int off, int len) {
+ if (buf.isDirect())
+ GridUnsafe.setMemory(GridUnsafe.bufferAddress(buf) + off, len, (byte)0);
+
+ else {
+ for (int i = off; i < off + len; i++)
+ buf.put(i, (byte)0); //TODO Optimize!
+ }
+ }
+
+ /**
+ * @param srcAddr Source.
+ * @param dstAddr Destination.
+ * @param srcOff Source offset in bytes.
+ * @param dstOff Destination offset in bytes.
+ * @param cnt Bytes count to copy.
+ */
+ public static void copyMemory(long srcAddr, long dstAddr, long srcOff, long dstOff, long cnt) {
+ GridUnsafe.copyMemory(null, srcAddr + srcOff, null, dstAddr + dstOff, cnt);
+ }
+
+ /**
+ * @param addr Address.
+ * @param off Offset.
+ * @param len Length.
+ */
+ public static void zeroMemory(long addr, int off, int len) {
+ GridUnsafe.setMemory(addr + off, len, (byte)0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageLockListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageLockListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageLockListener.java
new file mode 100644
index 0000000..75cf887
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/util/PageLockListener.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.tree.util;
+
+/**
+ * Page lock listener.
+ */
+public interface PageLockListener {
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ */
+ public void onBeforeWriteLock(int cacheId, long pageId, long page);
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ */
+ public void onWriteLock(int cacheId, long pageId, long page, long pageAddr);
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ */
+ public void onWriteUnlock(int cacheId, long pageId, long page, long pageAddr);
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ */
+ public void onBeforeReadLock(int cacheId, long pageId, long page);
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ */
+ public void onReadLock(int cacheId, long pageId, long page, long pageAddr);
+
+ /**
+ * @param cacheId Cache ID.
+ * @param pageId Page ID.
+ * @param page Page pointer.
+ * @param pageAddr Page address.
+ */
+ public void onReadUnlock(int cacheId, long pageId, long page, long pageAddr);
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInput.java
new file mode 100644
index 0000000..807a968
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInput.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+/**
+ * ByteBuffer backed data input
+ */
+public interface ByteBufferBackedDataInput extends DataInput {
+ /**
+ * @return ByteBuffer hold by data input
+ */
+ public ByteBuffer buffer();
+
+ /**
+ * ensure that requested count of byte is available in data input and will try to load data if not
+ * @param requested Requested number of bytes.
+ * @throws IOException If failed.
+ */
+ public void ensure(int requested) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInputImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInputImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInputImpl.java
new file mode 100644
index 0000000..2351ea7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/ByteBufferBackedDataInputImpl.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Byte buffer backed data input.
+ */
+public class ByteBufferBackedDataInputImpl implements ByteBufferBackedDataInput {
+ /** Buffer. */
+ private ByteBuffer buf;
+
+ /**
+ * @param buf New buffer.
+ */
+ public ByteBufferBackedDataInput buffer(ByteBuffer buf) {
+ this.buf = buf;
+
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override public ByteBuffer buffer() {
+ return buf;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void ensure(int requested) throws IOException {
+ if (buf.remaining() < requested)
+ throw new IOException("Requested size is greater than buffer: " + requested);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(@NotNull byte[] b) throws IOException {
+ ensure(b.length);
+
+ buf.get(b);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+ ensure(b.length);
+
+ buf.get(b, off, len);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int skipBytes(int n) throws IOException {
+ ensure(n);
+
+ buf.position(buf.position() + n);
+
+ return n;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public boolean readBoolean() throws IOException {
+ return readByte() == 1;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public byte readByte() throws IOException {
+ ensure(1);
+
+ return buf.get();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readUnsignedByte() throws IOException {
+ return readByte() & 0xFF;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public short readShort() throws IOException {
+ ensure(2);
+
+ return buf.getShort();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readUnsignedShort() throws IOException {
+ return readShort() & 0xFFFF;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public char readChar() throws IOException {
+ ensure(2);
+
+ return buf.getChar();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readInt() throws IOException {
+ ensure(4);
+
+ return buf.getInt();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public long readLong() throws IOException {
+ ensure(8);
+
+ return buf.getLong();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public float readFloat() throws IOException {
+ ensure(4);
+
+ return buf.getFloat();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public double readDouble() throws IOException {
+ ensure(8);
+
+ return buf.getDouble();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String readLine() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String readUTF() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
[14/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStoreManager.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStoreManager.java
deleted file mode 100755
index 8c096f1..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/file/FilePageStoreManager.java
+++ /dev/null
@@ -1,671 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.file;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.pagemem.store.PageStore;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedManagerAdapter;
-import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheSnapshotManager;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.marshaller.Marshaller;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
-import org.jetbrains.annotations.Nullable;
-
-/**
- * File page store manager.
- */
-public class FilePageStoreManager extends GridCacheSharedManagerAdapter implements IgnitePageStoreManager {
- /** File suffix. */
- public static final String FILE_SUFFIX = ".bin";
-
- /** Partition file prefix. */
- public static final String PART_FILE_PREFIX = "part-";
-
- /** */
- public static final String INDEX_FILE_NAME = "index" + FILE_SUFFIX;
-
- /** */
- public static final String PART_FILE_TEMPLATE = PART_FILE_PREFIX+ "%d" + FILE_SUFFIX;
-
- /** */
- public static final String CACHE_DIR_PREFIX = "cache-";
-
- /** */
- public static final String CACHE_GRP_DIR_PREFIX = "cacheGroup-";
-
- /** */
- public static final String CACHE_DATA_FILENAME = "cache_data.dat";
-
- /** Marshaller. */
- private static final Marshaller marshaller = new JdkMarshaller();
-
- /** */
- private final Map<Integer, CacheStoreHolder> idxCacheStores = new ConcurrentHashMap<>();
-
- /** */
- private final IgniteConfiguration igniteCfg;
-
- /** */
- private PersistentStoreConfiguration pstCfg;
-
- /** Absolute directory for file page store */
- private File storeWorkDir;
-
- /** */
- private final long metaPageId = PageIdUtils.pageId(-1, PageMemory.FLAG_IDX, 0);
-
- /** */
- private final Set<Integer> grpsWithoutIdx = Collections.newSetFromMap(new ConcurrentHashMap<Integer, Boolean>());
-
- /**
- * @param ctx Kernal context.
- */
- public FilePageStoreManager(GridKernalContext ctx) {
- igniteCfg = ctx.config();
-
- PersistentStoreConfiguration pstCfg = igniteCfg.getPersistentStoreConfiguration();
-
- assert pstCfg != null : "WAL should not be created if persistence is disabled.";
-
- this.pstCfg = pstCfg;
- }
-
- /** {@inheritDoc} */
- @Override public void start0() throws IgniteCheckedException {
- if (cctx.kernalContext().clientNode())
- return;
-
- String consId = U.maskForFileName(cctx.kernalContext().discovery().consistentId().toString());
-
- if (pstCfg.getPersistentStorePath() != null) {
- File workDir0 = new File(pstCfg.getPersistentStorePath());
-
- if (!workDir0.isAbsolute())
- workDir0 = U.resolveWorkDirectory(
- igniteCfg.getWorkDirectory(),
- pstCfg.getPersistentStorePath(),
- false
- );
-
- storeWorkDir = new File(workDir0, consId);
- }
- else
- storeWorkDir = new File(U.resolveWorkDirectory(
- igniteCfg.getWorkDirectory(),
- "db",
- false
- ), consId);
-
- U.ensureDirectory(storeWorkDir, "page store work directory", log);
- }
-
- /** {@inheritDoc} */
- @Override public void stop0(boolean cancel) {
- if (log.isDebugEnabled())
- log.debug("Stopping page store manager.");
-
- IgniteCheckedException ex = shutdown(false);
-
- if (ex != null)
- U.error(log, "Failed to gracefully stop page store manager", ex);
- }
-
- /** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Activate page store manager [id=" + cctx.localNodeId() +
- " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
-
- start0();
- }
-
- /** {@inheritDoc} */
- @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("DeActivate page store manager [id=" + cctx.localNodeId() +
- " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
-
- stop0(true);
-
- idxCacheStores.clear();
- }
-
- /** {@inheritDoc} */
- @Override public void beginRecover() {
- for (CacheStoreHolder holder : idxCacheStores.values()) {
- holder.idxStore.beginRecover();
-
- for (FilePageStore partStore : holder.partStores)
- partStore.beginRecover();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void finishRecover() {
- for (CacheStoreHolder holder : idxCacheStores.values()) {
- holder.idxStore.finishRecover();
-
- for (FilePageStore partStore : holder.partStores)
- partStore.finishRecover();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void initializeForCache(CacheGroupDescriptor grpDesc, StoredCacheData cacheData)
- throws IgniteCheckedException {
- int grpId = grpDesc.groupId();
-
- if (!idxCacheStores.containsKey(grpId)) {
- CacheStoreHolder holder = initForCache(grpDesc, cacheData.config());
-
- CacheStoreHolder old = idxCacheStores.put(grpId, holder);
-
- assert old == null : "Non-null old store holder for cache: " + cacheData.config().getName();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void storeCacheData(
- CacheGroupDescriptor grpDesc,
- StoredCacheData cacheData
- ) throws IgniteCheckedException {
-
- File cacheWorkDir = cacheWorkDirectory(grpDesc, cacheData.config());
- File file;
-
- checkAndInitCacheWorkDir(cacheWorkDir);
-
- assert cacheWorkDir.exists() : "Work directory does not exist: " + cacheWorkDir;
-
- if (grpDesc.sharedGroup())
- file = new File(cacheWorkDir, cacheData.config().getName() + CACHE_DATA_FILENAME);
- else
- file = new File(cacheWorkDir, CACHE_DATA_FILENAME);
-
- if (!file.exists() || file.length() == 0) {
- try {
- file.createNewFile();
-
- try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(file))) {
- marshaller.marshal(cacheData, stream);
- }
- }
- catch (IOException ex) {
- throw new IgniteCheckedException("Failed to persist cache configuration: " + cacheData.config().getName(), ex);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void shutdownForCacheGroup(CacheGroupContext grp, boolean destroy) throws IgniteCheckedException {
- grpsWithoutIdx.remove(grp.groupId());
-
- CacheStoreHolder old = idxCacheStores.remove(grp.groupId());
-
- assert old != null : "Missing cache store holder [cache=" + grp.cacheOrGroupName() +
- ", locNodeId=" + cctx.localNodeId() + ", gridName=" + cctx.igniteInstanceName() + ']';
-
- IgniteCheckedException ex = shutdown(old, /*clean files if destroy*/destroy, null);
-
- if (ex != null)
- throw ex;
- }
-
- /** {@inheritDoc} */
- @Override public void onPartitionCreated(int grpId, int partId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void onPartitionDestroyed(int grpId, int partId, int tag) throws IgniteCheckedException {
- assert partId <= PageIdAllocator.MAX_PARTITION_ID;
-
- PageStore store = getStore(grpId, partId);
-
- assert store instanceof FilePageStore : store;
-
- ((FilePageStore)store).truncate(tag);
- }
-
- /** {@inheritDoc} */
- @Override public void read(int cacheId, long pageId, ByteBuffer pageBuf) throws IgniteCheckedException {
- read(cacheId, pageId, pageBuf, false);
- }
-
- /**
- * Will preserve crc in buffer if keepCrc is true.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param pageBuf Page buffer.
- * @param keepCrc Keep CRC flag.
- * @throws IgniteCheckedException If failed.
- */
- public void read(int cacheId, long pageId, ByteBuffer pageBuf, boolean keepCrc) throws IgniteCheckedException {
- PageStore store = getStore(cacheId, PageIdUtils.partId(pageId));
-
- store.read(pageId, pageBuf, keepCrc);
- }
-
- /** {@inheritDoc} */
- @Override public boolean exists(int grpId, int partId) throws IgniteCheckedException {
- PageStore store = getStore(grpId, partId);
-
- return store.exists();
- }
-
- /** {@inheritDoc} */
- @Override public void readHeader(int cacheId, int partId, ByteBuffer buf) throws IgniteCheckedException {
- PageStore store = getStore(cacheId, partId);
-
- store.readHeader(buf);
- }
-
- /** {@inheritDoc} */
- @Override public void write(int cacheId, long pageId, ByteBuffer pageBuf,int tag) throws IgniteCheckedException {
- writeInternal(cacheId, pageId, pageBuf, tag);
- }
-
- /** {@inheritDoc} */
- @Override public long pageOffset(int cacheId, long pageId) throws IgniteCheckedException {
- PageStore store = getStore(cacheId, PageIdUtils.partId(pageId));
-
- return store.pageOffset(pageId);
- }
-
- /**
- * @param cacheId Cache ID to write.
- * @param pageId Page ID.
- * @param pageBuf Page buffer.
- * @return PageStore to which the page has been written.
- * @throws IgniteCheckedException If IO error occurred.
- */
- public PageStore writeInternal(int cacheId, long pageId, ByteBuffer pageBuf, int tag) throws IgniteCheckedException {
- int partId = PageIdUtils.partId(pageId);
-
- PageStore store = getStore(cacheId, partId);
-
- store.write(pageId, pageBuf, tag);
-
- return store;
- }
-
- /**
- * @param grpDesc Cache group descriptor.
- * @param ccfg Cache configuration.
- * @return Cache work directory.
- */
- private File cacheWorkDirectory(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) {
- String dirName;
-
- if (grpDesc.sharedGroup())
- dirName = CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
- else
- dirName = CACHE_DIR_PREFIX + ccfg.getName();
-
- return new File(storeWorkDir, dirName);
- }
-
- /**
- * @param grpDesc Cache group descriptor.
- * @param ccfg Cache configuration.
- * @return Cache store holder.
- * @throws IgniteCheckedException If failed.
- */
- private CacheStoreHolder initForCache(CacheGroupDescriptor grpDesc, CacheConfiguration ccfg) throws IgniteCheckedException {
- assert !grpDesc.sharedGroup() || ccfg.getGroupName() != null : ccfg.getName();
-
- File cacheWorkDir = cacheWorkDirectory(grpDesc, ccfg);
-
- boolean dirExisted = checkAndInitCacheWorkDir(cacheWorkDir);
-
- File idxFile = new File(cacheWorkDir, INDEX_FILE_NAME);
-
- if (dirExisted && !idxFile.exists())
- grpsWithoutIdx.add(grpDesc.groupId());
-
- FilePageStore idxStore = new FilePageStore(
- PageMemory.FLAG_IDX,
- idxFile,
- cctx.kernalContext().config().getMemoryConfiguration());
-
- FilePageStore[] partStores = new FilePageStore[grpDesc.config().getAffinity().partitions()];
-
- for (int partId = 0; partId < partStores.length; partId++) {
- FilePageStore partStore = new FilePageStore(
- PageMemory.FLAG_DATA,
- new File(cacheWorkDir, String.format(PART_FILE_TEMPLATE, partId)),
- cctx.kernalContext().config().getMemoryConfiguration()
- );
-
- partStores[partId] = partStore;
- }
-
- return new CacheStoreHolder(idxStore, partStores);
- }
-
- private boolean checkAndInitCacheWorkDir(File cacheWorkDir) throws IgniteCheckedException {
- boolean dirExisted = false;
-
- if (!cacheWorkDir.exists()) {
- boolean res = cacheWorkDir.mkdirs();
-
- if (!res)
- throw new IgniteCheckedException("Failed to initialize cache working directory " +
- "(failed to create, make sure the work folder has correct permissions): " +
- cacheWorkDir.getAbsolutePath());
- }
- else {
- if (cacheWorkDir.isFile())
- throw new IgniteCheckedException("Failed to initialize cache working directory " +
- "(a file with the same name already exists): " + cacheWorkDir.getAbsolutePath());
-
- File lockF = new File(cacheWorkDir, IgniteCacheSnapshotManager.SNAPSHOT_RESTORE_STARTED_LOCK_FILENAME);
-
- if (lockF.exists()) {
- Path tmp = cacheWorkDir.toPath().getParent().resolve(cacheWorkDir.getName() + ".tmp");
-
- boolean deleted = U.delete(cacheWorkDir);
-
- if (Files.exists(tmp) && Files.isDirectory(tmp)) {
- U.warn(log, "Ignite node crashed during the snapshot restore process " +
- "(there is a snapshot restore lock file left for cache). But old version of cache was saved. " +
- "Trying to restore it. Cache - [" + cacheWorkDir.getAbsolutePath() + ']');
-
- try {
- Files.move(tmp, cacheWorkDir.toPath());
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
- else {
- U.warn(log, "Ignite node crashed during the snapshot restore process " +
- "(there is a snapshot restore lock file left for cache). Will remove both the lock file and " +
- "incomplete cache directory [cacheDir=" + cacheWorkDir.getAbsolutePath() + ']');
-
- if (!deleted)
- throw new IgniteCheckedException("Failed to remove obsolete cache working directory " +
- "(remove the directory manually and make sure the work folder has correct permissions): " +
- cacheWorkDir.getAbsolutePath());
-
- cacheWorkDir.mkdirs();
- }
-
- if (!cacheWorkDir.exists())
- throw new IgniteCheckedException("Failed to initialize cache working directory " +
- "(failed to create, make sure the work folder has correct permissions): " +
- cacheWorkDir.getAbsolutePath());
- }
- else
- dirExisted = true;
- }
-
- return dirExisted;
- }
-
- /** {@inheritDoc} */
- @Override public void sync(int cacheId, int partId) throws IgniteCheckedException {
- getStore(cacheId, partId).sync();
- }
-
- /** {@inheritDoc} */
- @Override public void ensure(int cacheId, int partId) throws IgniteCheckedException {
- getStore(cacheId, partId).ensure();
- }
-
- /** {@inheritDoc} */
- @Override public long allocatePage(int cacheId, int partId, byte flags) throws IgniteCheckedException {
- assert partId <= PageIdAllocator.MAX_PARTITION_ID || partId == PageIdAllocator.INDEX_PARTITION;
-
- PageStore store = getStore(cacheId, partId);
-
- long pageIdx = store.allocatePage();
-
- return PageIdUtils.pageId(partId, flags, (int)pageIdx);
- }
-
- /** {@inheritDoc} */
- @Override public long metaPageId(final int cacheId) {
- return metaPageId;
- }
-
- /** {@inheritDoc} */
- @Override public int pages(int cacheId, int partId) throws IgniteCheckedException {
- PageStore store = getStore(cacheId, partId);
-
- return store.pages();
- }
-
- /** {@inheritDoc} */
- @Override public Map<String, StoredCacheData> readCacheConfigurations() throws IgniteCheckedException {
- if (cctx.kernalContext().clientNode())
- return Collections.emptyMap();
-
- File[] files = storeWorkDir.listFiles();
-
- if (files == null)
- return Collections.emptyMap();
-
- Map<String, StoredCacheData> ccfgs = new HashMap<>();
-
- for (File file : files) {
- if (file.isDirectory()) {
- if (file.getName().startsWith(CACHE_DIR_PREFIX)) {
- File conf = new File(file, CACHE_DATA_FILENAME);
-
- if (conf.exists() && conf.length() > 0) {
- StoredCacheData cacheData = readCacheData(conf);
-
- ccfgs.put(cacheData.config().getName(), cacheData);
- }
- }
- else if (file.getName().startsWith(CACHE_GRP_DIR_PREFIX))
- readCacheGroupCaches(file, ccfgs);
- }
- }
-
- return ccfgs;
- }
-
- /**
- * @param grpDir Group directory.
- * @param ccfgs Cache configurations.
- * @throws IgniteCheckedException If failed.
- */
- private void readCacheGroupCaches(File grpDir, Map<String, StoredCacheData> ccfgs) throws IgniteCheckedException {
- File[] files = grpDir.listFiles();
-
- if (files == null)
- return;
-
- for (File file : files) {
- if (!file.isDirectory() && file.getName().endsWith(CACHE_DATA_FILENAME) && file.length() > 0) {
- StoredCacheData cacheData = readCacheData(file);
-
- ccfgs.put(cacheData.config().getName(), cacheData);
- }
- }
- }
-
- /**
- * @param conf File with stored cache data.
- * @return Cache data.
- * @throws IgniteCheckedException If failed.
- */
- private StoredCacheData readCacheData(File conf) throws IgniteCheckedException {
- try (InputStream stream = new BufferedInputStream(new FileInputStream(conf))) {
- return marshaller.unmarshal(stream, U.resolveClassLoader(igniteCfg));
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to read cache configuration from disk for cache: " +
- conf.getAbsolutePath(), e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean hasIndexStore(int grpId) {
- return !grpsWithoutIdx.contains(grpId);
- }
-
- /**
- * @return Store work dir.
- */
- public File workDir() {
- return storeWorkDir;
- }
-
- /**
- * @param ccfg Cache configuration.
- * @return Store dir for given cache.
- */
- public File cacheWorkDir(CacheConfiguration ccfg) {
- String dirName = ccfg.getGroupName() == null ?
- CACHE_DIR_PREFIX + ccfg.getName() : CACHE_GRP_DIR_PREFIX + ccfg.getGroupName();
-
- return new File(storeWorkDir, dirName);
- }
-
- /**
- * @param cleanFiles {@code True} if the stores should delete it's files upon close.
- */
- private IgniteCheckedException shutdown(boolean cleanFiles) {
- IgniteCheckedException ex = null;
-
- for (CacheStoreHolder holder : idxCacheStores.values())
- ex = shutdown(holder, cleanFiles, ex);
-
- return ex;
- }
-
- /**
- * @param holder Store holder.
- * @param cleanFile {@code True} if files should be cleaned.
- * @param aggr Aggregating exception.
- * @return Aggregating exception, if error occurred.
- */
- private IgniteCheckedException shutdown(CacheStoreHolder holder, boolean cleanFile,
- @Nullable IgniteCheckedException aggr) {
- aggr = shutdown(holder.idxStore, cleanFile, aggr);
-
- for (FilePageStore store : holder.partStores) {
- if (store != null)
- aggr = shutdown(store, cleanFile, aggr);
- }
-
- return aggr;
- }
-
- /**
- * @param store Store to shutdown.
- * @param cleanFile {@code True} if files should be cleaned.
- * @param aggr Aggregating exception.
- * @return Aggregating exception, if error occurred.
- */
- private IgniteCheckedException shutdown(FilePageStore store, boolean cleanFile, IgniteCheckedException aggr) {
- try {
- if (store != null)
- store.stop(cleanFile);
- }
- catch (IgniteCheckedException e) {
- if (aggr == null)
- aggr = new IgniteCheckedException("Failed to gracefully shutdown store");
-
- aggr.addSuppressed(e);
- }
-
- return aggr;
- }
-
- /**
- * @param grpId Cache group ID.
- * @param partId Partition ID.
- * @return Page store for the corresponding parameters.
- * @throws IgniteCheckedException If cache or partition with the given ID was not created.
- *
- * Note: visible for testing.
- */
- public PageStore getStore(int grpId, int partId) throws IgniteCheckedException {
- CacheStoreHolder holder = idxCacheStores.get(grpId);
-
- if (holder == null)
- throw new IgniteCheckedException("Failed to get page store for the given cache ID " +
- "(cache has not been started): " + grpId);
-
- if (partId == PageIdAllocator.INDEX_PARTITION)
- return holder.idxStore;
-
- if (partId > PageIdAllocator.MAX_PARTITION_ID)
- throw new IgniteCheckedException("Partition ID is reserved: " + partId);
-
- FilePageStore store = holder.partStores[partId];
-
- if (store == null)
- throw new IgniteCheckedException("Failed to get page store for the given partition ID " +
- "(partition has not been created) [grpId=" + grpId + ", partId=" + partId + ']');
-
- return store;
- }
-
- /**
- *
- */
- private static class CacheStoreHolder {
- /** Index store. */
- private final FilePageStore idxStore;
-
- /** Partition stores. */
- private final FilePageStore[] partStores;
-
- /**
- *
- */
- public CacheStoreHolder(FilePageStore idxStore, FilePageStore[] partStores) {
- this.idxStore = idxStore;
- this.partStores = partStores;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/CheckpointMetricsTracker.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/CheckpointMetricsTracker.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/CheckpointMetricsTracker.java
deleted file mode 100644
index e6b88d3..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/CheckpointMetricsTracker.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.pagemem;
-
-import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-
-/**
- * Tracks various checkpoint phases and stats.
- *
- * Assumed sequence of events:
- * <ol>
- * <li>Checkpoint start</li>
- * <li>CP Lock wait start</li>
- * <li>CP mark start</li>
- * <li>CP Lock release</li>
- * <li>Pages write start</li>
- * <li>fsync start</li>
- * <li>Checkpoint end</li>
- * </ol>
- */
-public class CheckpointMetricsTracker {
- /** */
- private static final AtomicIntegerFieldUpdater<CheckpointMetricsTracker> DATA_PAGES_UPDATER =
- AtomicIntegerFieldUpdater.newUpdater(CheckpointMetricsTracker.class, "dataPages");
-
- /** */
- private static final AtomicIntegerFieldUpdater<CheckpointMetricsTracker> COW_PAGES_UPDATER =
- AtomicIntegerFieldUpdater.newUpdater(CheckpointMetricsTracker.class, "cowPages");
-
- /** */
- private volatile int dataPages;
-
- /** */
- private volatile int cowPages;
-
- /** */
- private long cpStart = System.currentTimeMillis();
-
- /** */
- private long cpLockWaitStart;
-
- /** */
- private long cpMarkStart;
-
- /** */
- private long cpLockRelease;
-
- /** */
- private long cpPagesWriteStart;
-
- /** */
- private long cpFsyncStart;
-
- /** */
- private long cpEnd;
-
- /**
- *
- */
- public void onCowPageWritten() {
- COW_PAGES_UPDATER.incrementAndGet(this);
- }
-
- /**
- *
- */
- public void onDataPageWritten() {
- DATA_PAGES_UPDATER.incrementAndGet(this);
- }
-
- /**
- * @return COW pages.
- */
- public int cowPagesWritten() {
- return cowPages;
- }
-
- /**
- * @return Data pages written.
- */
- public int dataPagesWritten() {
- return dataPages;
- }
-
- /**
- *
- */
- public void onLockWaitStart() {
- cpLockWaitStart = System.currentTimeMillis();
- }
-
- /**
- *
- */
- public void onMarkStart() {
- cpMarkStart = System.currentTimeMillis();
- }
-
- /**
- *
- */
- public void onLockRelease() {
- cpLockRelease = System.currentTimeMillis();
- }
-
- /**
- *
- */
- public void onPagesWriteStart() {
- cpPagesWriteStart = System.currentTimeMillis();
- }
-
- /**
- *
- */
- public void onFsyncStart() {
- cpFsyncStart = System.currentTimeMillis();
- }
-
- /**
- *
- */
- public void onEnd() {
- cpEnd = System.currentTimeMillis();
- }
-
- /**
- * @return Total checkpoint duration.
- */
- public long totalDuration() {
- return cpEnd - cpStart;
- }
-
- /**
- * @return Checkpoint lock wait duration.
- */
- public long lockWaitDuration() {
- return cpMarkStart - cpLockWaitStart;
- }
-
- /**
- * @return Checkpoint mark duration.
- */
- public long markDuration() {
- return cpPagesWriteStart - cpMarkStart;
- }
-
- /**
- * @return Checkpoint lock hold duration.
- */
- public long lockHoldDuration() {
- return cpLockRelease - cpMarkStart;
- }
-
- /**
- * @return Pages write duration.
- */
- public long pagesWriteDuration() {
- return cpFsyncStart - cpPagesWriteStart;
- }
-
- /**
- * @return Checkpoint fsync duration.
- */
- public long fsyncDuration() {
- return cpEnd - cpFsyncStart;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/EvictCandidate.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/EvictCandidate.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/EvictCandidate.java
deleted file mode 100644
index 906871f..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/EvictCandidate.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.pagemem;
-
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.util.tostring.GridToStringExclude;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class EvictCandidate {
- /** */
- private int tag;
-
- /** */
- @GridToStringExclude
- private long relPtr;
-
- /** */
- @GridToStringInclude
- private FullPageId fullId;
-
- /**
- * @param tag Tag.
- * @param relPtr Relative pointer.
- * @param fullId Full page ID.
- */
- public EvictCandidate(int tag, long relPtr, FullPageId fullId) {
- this.tag = tag;
- this.relPtr = relPtr;
- this.fullId = fullId;
- }
-
- /**
- * @return Tag.
- */
- public int tag() {
- return tag;
- }
-
- /**
- * @return Relative pointer.
- */
- public long relativePointer() {
- return relPtr;
- }
-
- /**
- * @return Index.
- */
- public FullPageId fullId() {
- return fullId;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(EvictCandidate.class, this, "relPtr", U.hexLong(relPtr));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/FullPageIdTable.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/FullPageIdTable.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/FullPageIdTable.java
deleted file mode 100644
index 01d7bce..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/FullPageIdTable.java
+++ /dev/null
@@ -1,556 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.pagemem;
-
-import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteBiInClosure;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR;
-import static org.apache.ignite.IgniteSystemProperties.getFloat;
-
-/**
- *
- */
-public class FullPageIdTable {
- /** Load factor. */
- private static final float LOAD_FACTOR = getFloat(IGNITE_LONG_LONG_HASH_MAP_LOAD_FACTOR, 2.5f);
-
- /** */
- private static final int BYTES_PER_ENTRY = /*cache ID*/4 + /*partition tag*/4 + /*page ID*/8 + /*pointer*/8;
-
- /** */
- private static final FullPageId EMPTY_FULL_PAGE_ID = new FullPageId(0, 0);
-
- /** */
- private static final long EMPTY_PAGE_ID = EMPTY_FULL_PAGE_ID.pageId();
-
- /** */
- private static final int EMPTY_CACHE_ID = EMPTY_FULL_PAGE_ID.cacheId();
-
- /** */
- private static final long REMOVED_PAGE_ID = 0x8000000000000000L;
-
- /** */
- private static final int REMOVED_CACHE_ID = 0;
-
- /** */
- private static final int EQUAL = 0;
-
- /** */
- private static final int EMPTY = 1;
-
- /** */
- private static final int REMOVED = -1;
-
- /** */
- private static final int NOT_EQUAL = 2;
-
- /** */
- private static final int OUTDATED = -3;
-
- /** Max size, in elements. */
- protected int capacity;
-
- /** Maximum number of steps to try before failing. */
- protected int maxSteps;
-
- /** Pointer to the values array. */
- protected long valPtr;
-
- /**
- * @return Estimated memory size required for this map to store the given number of elements.
- */
- public static long requiredMemory(long elementCnt) {
- assert LOAD_FACTOR != 0;
-
- return ((long)(elementCnt * LOAD_FACTOR)) * BYTES_PER_ENTRY + 8;
- }
-
- /**
- * @param addr Base address.
- * @param len Allocated memory length.
- * @param clear If {@code true}, then memory is considered dirty and will be cleared. Otherwise,
- * map will assume that the given memory region is in valid state.
- */
- public FullPageIdTable(long addr, long len, boolean clear) {
- valPtr = addr;
- capacity = (int)((len - 8) / BYTES_PER_ENTRY);
-
- maxSteps = capacity;
-
- if (clear)
- clear();
- }
-
- /**
- * @return Current number of entries in the map.
- */
- public final int size() {
- return GridUnsafe.getInt(valPtr);
- }
-
- /**
- * @return Maximum number of entries in the map. This maximum can not be always reached.
- */
- public final int capacity() {
- return capacity;
- }
-
- /**
- * Gets value associated with the given key.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return A value associated with the given key.
- */
- public long get(int cacheId, long pageId, int tag, long absent, long outdated) {
- assert assertKey(cacheId, pageId);
-
- int idx = getKey(cacheId, pageId, tag, false);
-
- if (idx == -1)
- return absent;
-
- if (idx == -2)
- return outdated;
-
- return valueAt(idx);
- }
-
- /**
- * Refresh outdated value.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param tag Partition tag.
- * @return A value associated with the given key.
- */
- public long refresh(int cacheId, long pageId, int tag) {
- assert assertKey(cacheId, pageId);
-
- int idx = getKey(cacheId, pageId, tag, true);
-
- assert idx >= 0 : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
- ", pageId=" + U.hexLong(pageId) + ']';
-
- assert tagAt(idx) < tag : "[idx=" + idx + ", tag=" + tag + ", cacheId=" + cacheId +
- ", pageId=" + U.hexLong(pageId) + ", tagAtIdx=" + tagAt(idx) + ']';
-
- setTagAt(idx, tag);
-
- return valueAt(idx);
- }
-
- /**
- * Associates the given key with the given value.
- * @param cacheId Cache ID
- * @param pageId Page ID.
- * @param value Value to set.
- */
- public void put(int cacheId, long pageId, long value, int tag) {
- assert assertKey(cacheId, pageId);
-
- int index = putKey(cacheId, pageId, tag);
-
- setValueAt(index, value);
- }
-
- /**
- * Removes key-value association for the given key.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- */
- public void remove(int cacheId, long pageId, int tag) {
- assert assertKey(cacheId, pageId);
-
- int index = removeKey(cacheId, pageId, tag);
-
- if (index >= 0)
- setValueAt(index, 0);
- }
-
- /**
- * Find nearest value from specified position to the right.
- *
- * @param idx Index to start searching from.
- * @param absent Default value that will be returned if no values present.
- * @return Closest value to the index and it's partition tag or {@code absent} and -1 if no values found.
- */
- public EvictCandidate getNearestAt(final int idx, final long absent) {
- for (int i = idx; i < capacity + idx; i++) {
- final int idx2 = i >= capacity ? i - capacity : i;
-
- if (isValuePresentAt(idx2)) {
- long base = entryBase(idx2);
-
- int cacheId = GridUnsafe.getInt(base);
- int tag = GridUnsafe.getInt(base + 4);
- long pageId = GridUnsafe.getLong(base + 8);
- long val = GridUnsafe.getLong(base + 16);
-
- return new EvictCandidate(tag, val, new FullPageId(pageId, cacheId));
- }
- }
-
- return null;
- }
-
- /**
- * @param idx Index to clear value at.
- * @param pred Test predicate.
- * @param absent Value to return if the cell is empty.
- * @return Value at the given index.
- */
- public long clearAt(int idx, GridPredicate3<Integer, Long, Integer> pred, long absent) {
- long base = entryBase(idx);
-
- int cacheId = GridUnsafe.getInt(base);
- int tag = GridUnsafe.getInt(base + 4);
- long pageId = GridUnsafe.getLong(base + 8);
-
- if ((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
- || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID))
- return absent;
-
- if (pred.apply(cacheId, pageId, tag)) {
- long res = valueAt(idx);
-
- setKeyAt(idx, REMOVED_CACHE_ID, REMOVED_PAGE_ID);
- setValueAt(idx, 0);
-
- return res;
- }
- else
- return absent;
- }
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return Key index.
- */
- private int putKey(int cacheId, long pageId, int tag) {
- int step = 1;
-
- int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
-
- int foundIndex = -1;
- int res;
-
- do {
- res = testKeyAt(index, cacheId, pageId, tag);
-
- if (res == OUTDATED) {
- foundIndex = index;
-
- break;
- }
- else if (res == EMPTY) {
- if (foundIndex == -1)
- foundIndex = index;
-
- break;
- }
- else if (res == REMOVED) {
- // Must continue search to the first empty slot to ensure there are no duplicate mappings.
- if (foundIndex == -1)
- foundIndex = index;
- }
- else if (res == EQUAL)
- return index;
- else
- assert res == NOT_EQUAL;
-
- index++;
-
- if (index >= capacity)
- index -= capacity;
- }
- while (++step <= maxSteps);
-
- if (foundIndex != -1) {
- setKeyAt(foundIndex, cacheId, pageId);
- setTagAt(foundIndex, tag);
-
- if (res != OUTDATED)
- incrementSize();
-
- return foundIndex;
- }
-
- throw new IgniteOutOfMemoryException("No room for a new key");
- }
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return Key index.
- */
- private int getKey(int cacheId, long pageId, int tag, boolean refresh) {
- int step = 1;
-
- int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
-
- do {
- long res = testKeyAt(index, cacheId, pageId, tag);
-
- if (res == EQUAL)
- return index;
- else if (res == EMPTY)
- return -1;
- else if (res == OUTDATED)
- return !refresh ? -2 : index;
- else
- assert res == REMOVED || res == NOT_EQUAL;
-
- index++;
-
- if (index >= capacity)
- index -= capacity;
- }
- while (++step <= maxSteps);
-
- return -1;
- }
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return Key index.
- */
- private int removeKey(int cacheId, long pageId, int tag) {
- int step = 1;
-
- int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
-
- int foundIndex = -1;
-
- do {
- long res = testKeyAt(index, cacheId, pageId, tag);
-
- if (res == EQUAL || res == OUTDATED) {
- foundIndex = index;
-
- break;
- }
- else if (res == EMPTY)
- return -1;
- else
- assert res == REMOVED || res == NOT_EQUAL;
-
- index++;
-
- if (index >= capacity)
- index -= capacity;
- }
- while (++step <= maxSteps);
-
- if (foundIndex != -1) {
- setKeyAt(foundIndex, REMOVED_CACHE_ID, REMOVED_PAGE_ID);
-
- decrementSize();
- }
-
- return foundIndex;
- }
-
- /**
- * @param index Entry index.
- * @return Key value.
- */
- private int testKeyAt(int index, int testCacheId, long testPageId, int testTag) {
- long base = entryBase(index);
-
- int cacheId = GridUnsafe.getInt(base);
- int tag = GridUnsafe.getInt(base + 4);
- long pageId = GridUnsafe.getLong(base + 8);
-
- if (pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
- return REMOVED;
- else if (pageId == testPageId && cacheId == testCacheId && tag >= testTag)
- return EQUAL;
- else if (pageId == testPageId && cacheId == testCacheId && tag < testTag)
- return OUTDATED;
- else if (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID)
- return EMPTY;
- else
- return NOT_EQUAL;
- }
-
- /**
- * @param idx Index to test.
- * @return {@code True} if value set for index.
- */
- private boolean isValuePresentAt(final int idx) {
- long base = entryBase(idx);
-
- int cacheId = GridUnsafe.getInt(base);
- long pageId = GridUnsafe.getLong(base + 8);
-
- return !((pageId == REMOVED_PAGE_ID && cacheId == REMOVED_CACHE_ID)
- || (pageId == EMPTY_PAGE_ID && cacheId == EMPTY_CACHE_ID));
- }
-
- /**
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @return {@code True} if checks succeeded.
- */
- private boolean assertKey(int cacheId, long pageId) {
- assert cacheId != EMPTY_CACHE_ID && PageIdUtils.isEffectivePageId(pageId):
- "cacheId=" + cacheId + ", pageId=" + U.hexLong(pageId);
-
- return true;
- }
-
- /**
- * @param index Entry index.
- * @param cacheId Cache ID to write.
- * @param pageId Page ID to write.
- */
- private void setKeyAt(int index, int cacheId, long pageId) {
- long base = entryBase(index);
-
- GridUnsafe.putInt(base, cacheId);
- GridUnsafe.putLong(base + 8, pageId);
- }
-
- /**
- * Gets distance from the ideal key location to the actual location if this entry is present in the table,
- * or returns negative distance that needed to be scanned to determine the absence of the mapping.
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param tag Tag.
- * @return Distance scanned if the entry is found or negative distance scanned, if entry was not found.
- */
- public int distanceFromIdeal(int cacheId, long pageId, int tag) {
- int step = 1;
-
- int index = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
-
- int scans = 0;
- boolean found = false;
-
- do {
- scans++;
-
- long res = testKeyAt(index, cacheId, pageId, tag);
-
- if (res == EQUAL || res == OUTDATED) {
- found = true;
-
- break;
- }
- else if (res == EMPTY)
- break;
- else
- assert res == REMOVED || res == NOT_EQUAL;
-
- index++;
-
- if (index >= capacity)
- index -= capacity;
- }
- while (++step <= maxSteps);
-
- return found ? scans : -scans;
- }
-
- /**
- * Scans all the elements in this table.
- *
- * @param visitor Visitor.
- */
- public void visitAll(IgniteBiInClosure<FullPageId, Long> visitor) {
- for (int i = 0; i < capacity; i++) {
- if (isValuePresentAt(i)) {
- long base = entryBase(i);
-
- int cacheId = GridUnsafe.getInt(base);
- long pageId = GridUnsafe.getLong(base + 8);
- long val = GridUnsafe.getLong(base + 16);
-
- visitor.apply(new FullPageId(pageId, cacheId), val);
- }
- }
- }
-
- /**
- * @param index Entry index.
- * @return Value.
- */
- private long valueAt(int index) {
- return GridUnsafe.getLong(entryBase(index) + 16);
- }
-
- /**
- * @param index Entry index.
- * @param value Value.
- */
- private void setValueAt(int index, long value) {
- GridUnsafe.putLong(entryBase(index) + 16, value);
- }
-
- private long entryBase(int index) {
- return valPtr + 8 + (long)index * BYTES_PER_ENTRY;
- }
-
- /**
- * @param index Index to get tag for.
- * @return Tag at the given index.
- */
- private int tagAt(int index) {
- return GridUnsafe.getInt(entryBase(index) + 4);
- }
-
- /**
- * @param index Index to set tag for.
- * @param tag Tag to set at the given index.
- */
- private void setTagAt(int index, int tag) {
- GridUnsafe.putInt(entryBase(index) + 4, tag);
- }
-
- /**
- *
- */
- public void clear() {
- GridUnsafe.setMemory(valPtr, (long)capacity * BYTES_PER_ENTRY + 8, (byte)0);
- }
-
- /**
- *
- */
- private void incrementSize() {
- GridUnsafe.putInt(valPtr, GridUnsafe.getInt(valPtr) + 1);
- }
-
- /**
- *
- */
- private void decrementSize() {
- GridUnsafe.putInt(valPtr, GridUnsafe.getInt(valPtr) - 1);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryEx.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryEx.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryEx.java
deleted file mode 100644
index 56d1be9..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/pagemem/PageMemoryEx.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.pagemem;
-
-import java.nio.ByteBuffer;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
-import org.apache.ignite.internal.util.lang.GridPredicate3;
-import org.jetbrains.annotations.Nullable;
-
-/**
- *
- */
-public interface PageMemoryEx extends PageMemory {
- /**
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param restore Determines if the page is locked for restore.
- * @return ByteBuffer for modifying the page.
- */
- long writeLock(int cacheId, long pageId, long page, boolean restore);
-
- /**
- *
- * @param cacheId Cache ID.
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param walPlc {@code True} if page should be recorded to WAL, {@code false} if the page must not
- * be recorded and {@code null} for the default behavior.
- * @param dirtyFlag Determines whether the page was modified since the last checkpoint.
- * @param restore Determines if the page is locked for restore.
- */
- void writeUnlock(int cacheId, long pageId, long page, Boolean walPlc,
- boolean dirtyFlag, boolean restore);
-
- /**
- * Gets or allocates metadata page for specified cacheId.
- *
- * @param cacheId Cache ID.
- * @return Meta page for cacheId.
- * @throws IgniteCheckedException If failed.
- */
- public long metaPageId(int cacheId) throws IgniteCheckedException;
-
- /**
- * Gets or allocates partition metadata page for specified cacheId and partId.
- *
- * @param cacheId Cache ID.
- * @param partId Partition ID.
- * @return Meta page for cacheId and partId.
- * @throws IgniteCheckedException If failed.
- */
- public long partitionMetaPageId(int cacheId, int partId) throws IgniteCheckedException;
-
- /**
- * @see #acquirePage(int, long)
- * Will not read page from file if it is not present in memory
- *
- * @param cacheId Cache id.
- * @param pageId Page id.
- * @param restore Get page for restore
- * @throws IgniteCheckedException If failed.
- * @return Page.
- */
- public long acquirePage(int cacheId, long pageId, boolean restore) throws IgniteCheckedException;
-
- /**
- * Heuristic method which allows a thread to check if it safe to start memory struture modifications
- * in regard with checkpointing.
- *
- * @return {@code False} if there are too many dirty pages and a thread should wait for a
- * checkpoint to begin.
- */
- public boolean safeToUpdate();
-
- /**
- * Gets a collection of dirty page IDs since the last checkpoint. If a dirty page is being written after
- * the checkpointing operation begun, the modifications will be written to a temporary buffer which will
- * be flushed to the main memory after the checkpointing finished. This method must be called when no
- * concurrent operations on pages are performed.
- *
- * @return Collection of dirty page IDs.
- * @throws IgniteException If checkpoint has been already started and was not finished.
- */
- public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException;
-
- /**
- * Finishes checkpoint operation.
- */
- public void finishCheckpoint();
-
- /**
- * Gets page byte buffer for the checkpoint procedure.
- *
- * @param pageId Page ID to get byte buffer for. The page ID must be present in the collection returned by
- * the {@link #beginCheckpoint()} method call.
- * @param tmpBuf Temporary buffer to write changes into.
- * @param tracker Checkpoint metrics tracker.
- * @return {@code True} if data were read, {@code false} otherwise (data already saved to storage).
- * @throws IgniteException If failed to obtain page data.
- */
- @Nullable public Integer getForCheckpoint(FullPageId pageId, ByteBuffer tmpBuf, CheckpointMetricsTracker tracker);
-
- /**
- * Marks partition as invalid / outdated.
- *
- * @param cacheId Cache ID.
- * @param partId Partition ID.
- * @return New partition tag.
- */
- public int invalidate(int cacheId, int partId);
-
- /**
- * Clears internal metadata of destroyed cache group.
- *
- * @param grpId Cache group ID.
- */
- public void onCacheGroupDestroyed(int grpId);
-
- /**
- * Asynchronously clears pages satisfying the given predicate.
- *
- * @param pred Predicate for cache group id, pageId and partition tag.
- * @param cleanDirty Flag indicating that dirty pages collection should be cleaned.
- * @return Future that will be completed when all pages are cleared.
- */
- public IgniteInternalFuture<Void> clearAsync(GridPredicate3<Integer, Long, Integer> pred, boolean cleanDirty);
-}
[03/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStructureTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStructureTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStructureTest.java
deleted file mode 100644
index d10521e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateDataStructureTest.java
+++ /dev/null
@@ -1,278 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteAtomicLong;
-import org.apache.ignite.IgniteAtomicSequence;
-import org.apache.ignite.IgniteCountDownLatch;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-import static org.apache.ignite.testframework.GridTestUtils.runAsync;
-
-/**
- *
- */
-public class IgniteChangeGlobalStateDataStructureTest extends IgniteChangeGlobalStateAbstractTest {
- /**
- *
- */
- public void testDeActivateAndActivateAtomicLong() throws Exception{
- String lName = "myLong";
-
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- IgniteAtomicLong lexp = ig1.atomicLong(lName, 100, true);
-
- lexp.incrementAndGet();
- lexp.incrementAndGet();
- lexp.incrementAndGet();
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig2.active(false);
-
- IgniteEx ex1 = (IgniteEx)ig1;
- IgniteEx ex2 = (IgniteEx)ig2;
- IgniteEx ex3 = (IgniteEx)ig3;
-
- GridCacheProcessor cache1 = ex1.context().cache();
- GridCacheProcessor cache2 = ex2.context().cache();
- GridCacheProcessor cache3 = ex3.context().cache();
-
- assertTrue(F.isEmpty(cache1.jcaches()));
- assertTrue(F.isEmpty(cache2.jcaches()));
- assertTrue(F.isEmpty(cache3.jcaches()));
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- IgniteAtomicLong lact1 = ig1.atomicLong(lName, 0, false);
- IgniteAtomicLong lact2 = ig2.atomicLong(lName, 0, false);
- IgniteAtomicLong lact3 = ig3.atomicLong(lName, 0, false);
-
- assertEquals(103, lact1.get());
- assertEquals(103, lact2.get());
- assertEquals(103, lact3.get());
-
- lact1.incrementAndGet();
-
- assertEquals(104, lact1.get());
- assertEquals(104, lact2.get());
- assertEquals(104, lact3.get());
-
- lact2.incrementAndGet();
-
- assertEquals(105, lact1.get());
- assertEquals(105, lact2.get());
- assertEquals(105, lact3.get());
-
- lact3.incrementAndGet();
-
- assertEquals(106, lact3.get());
- assertEquals(106, lact3.get());
- assertEquals(106, lact3.get());
- }
-
- /**
- *
- */
- public void testDeActivateAndActivateCountDownLatch() throws Exception {
- final AtomicInteger cnt = new AtomicInteger();
-
- String latchName = "myLatch";
-
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- IgniteCountDownLatch latchExp1 = ig1.countDownLatch(latchName, 5, false, true);
-
- latchExp1.countDown();
-
- assertEquals(4, latchExp1.count());
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig2.active(false);
-
- IgniteEx ex1 = (IgniteEx)ig1;
- IgniteEx ex2 = (IgniteEx)ig2;
- IgniteEx ex3 = (IgniteEx)ig3;
-
- GridCacheProcessor cache1 = ex1.context().cache();
- GridCacheProcessor cache2 = ex2.context().cache();
- GridCacheProcessor cache3 = ex3.context().cache();
-
- assertTrue(F.isEmpty(cache1.jcaches()));
- assertTrue(F.isEmpty(cache2.jcaches()));
- assertTrue(F.isEmpty(cache3.jcaches()));
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- final IgniteCountDownLatch latchAct1 = ig1.countDownLatch(latchName, 0, false, false);
- final IgniteCountDownLatch latchAct2 = ig2.countDownLatch(latchName, 0, false, false);
- final IgniteCountDownLatch latchAct3 = ig3.countDownLatch(latchName, 0, false, false);
-
- runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- latchAct1.await();
- cnt.incrementAndGet();
- return null;
- }
- });
-
- runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- latchAct2.await();
- cnt.incrementAndGet();
- return null;
- }
- });
-
- runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- latchAct3.await();
- cnt.incrementAndGet();
- return null;
- }
- });
-
-
- assertEquals(4, latchAct1.count());
- assertEquals(4, latchAct2.count());
- assertEquals(4, latchAct3.count());
-
- latchAct1.countDown();
- latchAct2.countDown();
- latchAct3.countDown();
-
- assertEquals(1, latchAct1.count());
- assertEquals(1, latchAct2.count());
- assertEquals(1, latchAct3.count());
-
- latchAct1.countDown();
-
- U.sleep(3000);
-
- assertEquals(3, cnt.get());
- }
-
- /**
- *
- */
- public void testDeActivateAndActivateAtomicSequence(){
- String seqName = "mySeq";
-
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- IgniteAtomicSequence seqExp1 = ig1.atomicSequence(seqName, 0, true);
- IgniteAtomicSequence seqExp2 = ig2.atomicSequence(seqName, 0, false);
- IgniteAtomicSequence seqExp3 = ig3.atomicSequence(seqName, 0, false);
-
- assertEquals(0, seqExp1.get());
- assertEquals(1000, seqExp2.get());
- assertEquals(2000, seqExp3.get());
-
- seqExp1.incrementAndGet();
- seqExp2.incrementAndGet();
- seqExp3.incrementAndGet();
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig2.active(false);
-
- IgniteEx ex1 = (IgniteEx)ig1;
- IgniteEx ex2 = (IgniteEx)ig2;
- IgniteEx ex3 = (IgniteEx)ig3;
-
- GridCacheProcessor cache1 = ex1.context().cache();
- GridCacheProcessor cache2 = ex2.context().cache();
- GridCacheProcessor cache3 = ex3.context().cache();
-
- assertTrue(F.isEmpty(cache1.jcaches()));
- assertTrue(F.isEmpty(cache2.jcaches()));
- assertTrue(F.isEmpty(cache3.jcaches()));
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- IgniteAtomicSequence seqAct1 = ig1.atomicSequence(seqName, 0, false);
- IgniteAtomicSequence seqAct2 = ig2.atomicSequence(seqName, 0, false);
- IgniteAtomicSequence seqAct3 = ig3.atomicSequence(seqName, 0, false);
-
- assertEquals(1, seqAct1.get());
- assertEquals(1001, seqAct2.get());
- assertEquals(2001, seqAct3.get());
-
- seqAct1.incrementAndGet();
-
- assertEquals(2, seqAct1.get());
- assertEquals(1001, seqAct2.get());
- assertEquals(2001, seqAct3.get());
-
- seqAct2.incrementAndGet();
-
- assertEquals(2, seqAct1.get());
- assertEquals(1002, seqAct2.get());
- assertEquals(2001, seqAct3.get());
-
- seqAct3.incrementAndGet();
-
- assertEquals(2, seqAct1.get());
- assertEquals(1002, seqAct2.get());
- assertEquals(2002, seqAct3.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateFailOverTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateFailOverTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateFailOverTest.java
deleted file mode 100644
index 53a2eec..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateFailOverTest.java
+++ /dev/null
@@ -1,353 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.internal.IgniteInternalFuture;
-
-import static java.lang.Thread.sleep;
-import static org.apache.ignite.testframework.GridTestUtils.runAsync;
-
-/**
- *
- */
-public class IgniteChangeGlobalStateFailOverTest extends IgniteChangeGlobalStateAbstractTest {
- /** {@inheritDoc} */
- @Override protected int primaryNodes() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override protected int primaryClientNodes() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override protected int backUpClientNodes() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override protected int backUpNodes() {
- return 4;
- }
-
- /**
- *
- */
- public void testActivateDeActivateOnFixTopology() throws Exception {
- final Ignite igB1 = backUp(0);
- final Ignite igB2 = backUp(1);
- final Ignite igB3 = backUp(2);
-
- assertTrue(!igB1.active());
- assertTrue(!igB2.active());
- assertTrue(!igB3.active());
-
- final AtomicInteger cntA = new AtomicInteger();
- final AtomicInteger cntD = new AtomicInteger();
-
- final AtomicLong timeA = new AtomicLong();
- final AtomicLong timeD = new AtomicLong();
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- final AtomicBoolean canAct = new AtomicBoolean(true);
-
- try {
- final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get()) {
- Ignite ig = randomBackUp(false);
-
- if (canAct.get()) {
- long start = System.currentTimeMillis();
-
- ig.active(true);
-
- timeA.addAndGet((System.currentTimeMillis() - start));
-
- cntA.incrementAndGet();
-
- for (Ignite ign : allBackUpNodes())
- assertTrue(ign.active());
-
- canAct.set(false);
- }
-
- }
- return null;
- }
- });
-
- final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get()) {
- Ignite ig = randomBackUp(false);
-
- if (!canAct.get()) {
- long start = System.currentTimeMillis();
-
- ig.active(false);
-
- timeD.addAndGet((System.currentTimeMillis() - start));
-
- cntD.incrementAndGet();
-
- for (Ignite ign : allBackUpNodes())
- assertTrue(!ign.active());
-
- canAct.set(true);
- }
-
- }
- return null;
- }
- });
-
- sleep(60_000);
-
- stop.set(true);
-
- af.get();
- df.get();
- }
- finally {
- log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:"
- + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()) + " nodes: " + backUpNodes()));
- }
- }
-
- /**
- *
- */
- public void testActivateDeActivateOnJoiningNode() throws Exception {
- final Ignite igB1 = backUp(0);
- final Ignite igB2 = backUp(1);
- final Ignite igB3 = backUp(2);
-
- assertTrue(!igB1.active());
- assertTrue(!igB2.active());
- assertTrue(!igB3.active());
-
- final AtomicInteger cntA = new AtomicInteger();
- final AtomicInteger cntD = new AtomicInteger();
-
- final AtomicLong timeA = new AtomicLong();
- final AtomicLong timeD = new AtomicLong();
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- final AtomicBoolean canAct = new AtomicBoolean(true);
- final AtomicInteger seqIdx = new AtomicInteger(backUpNodes());
-
- try {
- final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get()) {
- Ignite ig = randomBackUp(false);
-
- if (canAct.get()) {
- long start = System.currentTimeMillis();
-
- ig.active(true);
-
- timeA.addAndGet((System.currentTimeMillis() - start));
-
- cntA.incrementAndGet();
-
- for (Ignite ign : allBackUpNodes())
- assertTrue(ign.active());
-
- canAct.set(false);
- }
-
- }
- return null;
- }
- });
-
- final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get()) {
- Ignite ig = randomBackUp(false);
-
- if (!canAct.get()) {
- long start = System.currentTimeMillis();
-
- ig.active(false);
-
- timeD.addAndGet((System.currentTimeMillis() - start));
-
- cntD.incrementAndGet();
-
- for (Ignite ign : allBackUpNodes())
- assertTrue(!ign.active());
-
- canAct.set(true);
- }
-
- }
- return null;
- }
- });
-
- IgniteInternalFuture<Void> jf1 = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get())
- startBackUp(seqIdx.incrementAndGet());
-
- return null;
- }
- });
-
- IgniteInternalFuture<Void> jf2 = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get())
- startBackUp(seqIdx.incrementAndGet());
-
- return null;
- }
- });
-
- sleep(60_000);
-
- stop.set(true);
-
- af.get();
- df.get();
- jf1.get();
- jf2.get();
- }
- finally {
- log.info("total started nodes: " + (seqIdx.get() - backUpNodes()));
-
- log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime: "
- + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()))
- );
- }
- }
-
- /**
- *
- */
- public void testActivateDeActivateOnFixTopologyWithPutValues() throws Exception {
- final Ignite igB1 = backUp(0);
- final Ignite igB2 = backUp(1);
- final Ignite igB3 = backUp(2);
-
- assertTrue(!igB1.active());
- assertTrue(!igB2.active());
- assertTrue(!igB3.active());
-
- final CacheConfiguration<String, String> ccfg = new CacheConfiguration<>();
- ccfg.setName("main-cache");
-
- final AtomicInteger cnt = new AtomicInteger();
-
- final AtomicInteger cntA = new AtomicInteger();
- final AtomicInteger cntD = new AtomicInteger();
-
- final AtomicLong timeA = new AtomicLong();
- final AtomicLong timeD = new AtomicLong();
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- final AtomicBoolean canAct = new AtomicBoolean(true);
-
- try {
- final IgniteInternalFuture<Void> af = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get()) {
- Ignite ig = randomBackUp(false);
-
- if (canAct.get()) {
- long start = System.currentTimeMillis();
-
- ig.active(true);
-
- IgniteCache<String, String> cache = ig.getOrCreateCache(ccfg);
-
- cache.put("key" + cnt.get(), "value" + cnt.get());
-
- cnt.incrementAndGet();
-
- timeA.addAndGet((System.currentTimeMillis() - start));
-
- cntA.incrementAndGet();
-
- for (Ignite ign : allBackUpNodes())
- assertTrue(ign.active());
-
- canAct.set(false);
- }
-
- }
- return null;
- }
- });
-
- final IgniteInternalFuture<Void> df = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- while (!stop.get()) {
- Ignite ig = randomBackUp(false);
-
- if (!canAct.get()) {
- long start = System.currentTimeMillis();
-
- IgniteCache<String, String> cache = ig.getOrCreateCache(ccfg);
-
- for (int i = 0; i < cnt.get(); i++)
- assertEquals("value" + i, cache.get("key" + i));
-
- ig.active(false);
-
- timeD.addAndGet((System.currentTimeMillis() - start));
-
- cntD.incrementAndGet();
-
- for (Ignite ign : allBackUpNodes())
- assertTrue(!ign.active());
-
- canAct.set(true);
- }
-
- }
- return null;
- }
- });
-
- sleep(60_000);
-
- stop.set(true);
-
- af.get();
- df.get();
- }
- finally {
- log.info("total activate/deactivate:" + cntA.get() + "/" + cntD.get() + " aTime/dTime:"
- + (timeA.get() / cntA.get() + "/" + (timeD.get() / cntD.get()) + " nodes: " + backUpNodes()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateServiceTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateServiceTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateServiceTest.java
deleted file mode 100644
index da4d20c..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateServiceTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.Collection;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.services.Service;
-import org.apache.ignite.services.ServiceConfiguration;
-import org.apache.ignite.services.ServiceContext;
-import org.apache.ignite.services.ServiceDescriptor;
-
-/**
- *
- */
-public class IgniteChangeGlobalStateServiceTest extends IgniteChangeGlobalStateAbstractTest {
- /** {@inheritDoc} */
- @Override protected int backUpClientNodes() {
- return 0;
- }
-
- /** {@inheritDoc} */
- @Override protected int primaryNodes() {
- return 1;
- }
-
- /** {@inheritDoc} */
- @Override protected int backUpNodes() {
- return 1;
- }
-
- /**
- *
- */
- public void testDeployService() throws Exception {
- Ignite ig1P = primary(0);
-
- Ignite ig1B = backUp(0);
-
- String serName = "service";
-
- ServiceConfiguration serConf = new ServiceConfiguration();
- serConf.setTotalCount(1);
- serConf.setName(serName);
- serConf.setService(new TestService());
-
- ig1P.services().deploy(serConf);
-
- stopAllPrimary();
-
- ig1B.active(true);
-
- U.sleep(3000);
-
- Collection<ServiceDescriptor> descs = ig1B.services().serviceDescriptors();
-
- assertTrue(!F.isEmpty(descs));
-
- TestService srv = ig1B.services().service(serName);
-
- assertTrue(srv != null);
- }
-
- /**
- *
- */
- private static class TestService implements Service{
- /** */
- private static final long serialVersionUID = 0L;
-
- /** {@inheritDoc} */
- @Override public void cancel(ServiceContext ctx) {
- System.out.println("cancel service");
- }
-
- /** {@inheritDoc} */
- @Override public void init(ServiceContext ctx) throws Exception {
- System.out.println("init service");
- }
-
- /** {@inheritDoc} */
- @Override public void execute(ServiceContext ctx) throws Exception {
- System.out.println("execute service");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateTest.java
deleted file mode 100644
index d87ec5e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteChangeGlobalStateTest.java
+++ /dev/null
@@ -1,711 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteAtomicSequence;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteServices;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.util.typedef.F;
-
-import static org.apache.ignite.testframework.GridTestUtils.assertThrows;
-import static org.apache.ignite.testframework.GridTestUtils.runAsync;
-
-/**
- *
- */
-public class IgniteChangeGlobalStateTest extends IgniteChangeGlobalStateAbstractTest {
- /**
- * @throws Exception if fail.
- */
- public void testStopPrimaryAndActivateFromServerNode() throws Exception {
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- assertTrue(ig1P.active());
- assertTrue(ig2P.active());
- assertTrue(ig3P.active());
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- stopAllPrimary();
-
- ig2B.active(true);
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testStopPrimaryAndActivateFromClientNode() throws Exception {
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1C = backUpClient(0);
- Ignite ig2C = backUpClient(1);
- Ignite ig3C = backUpClient(2);
-
- assertTrue(ig1P.active());
- assertTrue(ig2P.active());
- assertTrue(ig3P.active());
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- stopAllPrimary();
-
- ig2B.active(true);
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
-
- assertTrue(ig1C.active());
- assertTrue(ig2C.active());
- assertTrue(ig3C.active());
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testConcurrentActivateFromClientNodeAndServerNode() throws Exception {
- final Ignite ig1B = backUp(0);
- final Ignite ig2B = backUp(1);
- final Ignite ig3B = backUp(2);
-
- final Ignite ig1C = backUpClient(0);
- final Ignite ig2C = backUpClient(1);
- final Ignite ig3C = backUpClient(2);
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- stopAllPrimary();
-
- final AtomicInteger exCnt = new AtomicInteger();
-
- final CyclicBarrier barrier = new CyclicBarrier(backUpNodes() + backUpClientNodes());
-
- IgniteInternalFuture<Void> f1 = runAsync(new Callable<Void>() {
- @Override public Void call() throws Exception {
- try {
- barrier.await();
- ig1B.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- return null;
- }
- });
-
- IgniteInternalFuture<Void> f2 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig2B.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- IgniteInternalFuture<Void> f3 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig3B.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- IgniteInternalFuture<Void> f4 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig1C.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- IgniteInternalFuture<Void> f5 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig2C.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- IgniteInternalFuture<Void> f6 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig3C.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- f1.get();
- f2.get();
- f3.get();
-
- f4.get();
- f5.get();
- f6.get();
-
- assertTrue(exCnt.get() > 0);
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
-
- assertTrue(ig1C.active());
- assertTrue(ig2C.active());
- assertTrue(ig3C.active());
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testConcurrentActivateFromServerNode() throws Exception {
- final Ignite ig1B = backUp(0);
- final Ignite ig2B = backUp(1);
- final Ignite ig3B = backUp(2);
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- stopAllPrimary();
-
- final AtomicInteger exCnt = new AtomicInteger();
-
- final CyclicBarrier barrier = new CyclicBarrier(3);
-
- IgniteInternalFuture<Void> act1 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig1B.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- IgniteInternalFuture<Void> act2 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig2B.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- IgniteInternalFuture<Void> act3 = runAsync(new Runnable() {
- @Override public void run() {
- try {
- barrier.await();
- ig3B.active(true);
- }catch (Exception e){
- exCnt.incrementAndGet();
- }
- }
- });
-
- act1.get();
- act2.get();
- act3.get();
-
- assertEquals(2, exCnt.get());
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
- }
-
- /**
- *
- */
- public void testActiveAndInActiveAtTheSameTimeCluster() throws Exception {
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- assertTrue(ig1P.cluster().nodes().size() == 6);
- assertTrue(ig2P.cluster().nodes().size() == 6);
- assertTrue(ig3P.cluster().nodes().size() == 6);
-
- List<ClusterNode> primaryNodes = Arrays.asList(
- ig1P.cluster().localNode(), ig2P.cluster().localNode(), ig3P.cluster().localNode()
- );
-
- List<ClusterNode> backUpNodes = Arrays.asList(
- ig1B.cluster().localNode(), ig3B.cluster().localNode(), ig3B.cluster().localNode()
- );
-
- assertTrue(!ig1P.cluster().forRemotes().nodes().containsAll(backUpNodes));
- assertTrue(!ig2P.cluster().forRemotes().nodes().containsAll(backUpNodes));
- assertTrue(!ig3P.cluster().forRemotes().nodes().containsAll(backUpNodes));
-
- assertTrue(ig1B.cluster().nodes().size() == 6);
- assertTrue(ig2B.cluster().nodes().size() == 6);
- assertTrue(ig3B.cluster().nodes().size() == 6);
-
- assertTrue(!ig1B.cluster().forRemotes().nodes().containsAll(primaryNodes));
- assertTrue(!ig2B.cluster().forRemotes().nodes().containsAll(primaryNodes));
- assertTrue(!ig3B.cluster().forRemotes().nodes().containsAll(primaryNodes));
- }
-
- /**
- *
- */
- public void testActivateOnAlreadyActivatedCluster() throws Exception {
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1C = backUpClient(0);
- Ignite ig2C = backUpClient(1);
- Ignite ig3C = backUpClient(2);
-
- assertTrue(ig1P.active());
- assertTrue(ig2P.active());
- assertTrue(ig3P.active());
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- stopAllPrimary();
-
- ig2B.active(true);
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
-
- assertTrue(ig1C.active());
- assertTrue(ig2C.active());
- assertTrue(ig3C.active());
-
- ig1B.active(true);
- ig2B.active(true);
- ig3B.active(true);
-
- ig1C.active(true);
- ig2C.active(true);
- ig3C.active(true);
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
-
- assertTrue(ig1C.active());
- assertTrue(ig2C.active());
- assertTrue(ig3C.active());
- }
-
- /**
- *
- */
- public void testTryUseCacheInActiveCluster() throws Exception {
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1C = backUpClient(0);
- Ignite ig2C = backUpClient(1);
- Ignite ig3C = backUpClient(2);
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- checkExceptionTryUseCache(ig1B);
- checkExceptionTryUseCache(ig2B);
- checkExceptionTryUseCache(ig3B);
-
- checkExceptionTryUseCache(ig1C);
- checkExceptionTryUseCache(ig2C);
- checkExceptionTryUseCache(ig3C);
- }
-
- /**
- * @param ig Ig.
- */
- private void checkExceptionTryUseCache(final Ignite ig) {
- assertThrows(log, new Callable<Void>() {
- @Override public Void call() throws Exception {
- IgniteCache cache = ig.cache("cache");
- return null;
- }
- }, IgniteException.class, "can not perform operation, because cluster inactive");
- }
-
- /**
- *
- */
- public void testTryUseServiceInActiveCluster() throws Exception {
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1C = backUpClient(0);
- Ignite ig2C = backUpClient(1);
- Ignite ig3C = backUpClient(2);
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- checkExceptionTryUseService(ig1B);
- checkExceptionTryUseService(ig2B);
- checkExceptionTryUseService(ig3B);
-
- checkExceptionTryUseService(ig1C);
- checkExceptionTryUseService(ig2C);
- checkExceptionTryUseService(ig3C);
- }
-
- /**
- *
- */
- private void checkExceptionTryUseService(final Ignite ig) {
- assertThrows(log, new Callable<Void>() {
- @Override public Void call() throws Exception {
- IgniteServices srvs = ig.services();
-
- return null;
- }
- }, IgniteException.class, "can not perform operation, because cluster inactive");
- }
-
- /**
- *
- */
- public void testTryUseDataStructureInActiveCluster() throws Exception {
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1C = backUpClient(0);
- Ignite ig2C = backUpClient(1);
- Ignite ig3C = backUpClient(2);
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- checkExceptionTryUseDataStructure(ig1B);
- checkExceptionTryUseDataStructure(ig2B);
- checkExceptionTryUseDataStructure(ig3B);
-
- checkExceptionTryUseDataStructure(ig1C);
- checkExceptionTryUseDataStructure(ig2C);
- checkExceptionTryUseDataStructure(ig3C);
- }
-
- /**
- *
- */
- private void checkExceptionTryUseDataStructure(final Ignite ig){
- assertThrows(log, new Callable<Void>() {
- @Override public Void call() throws Exception {
- IgniteAtomicSequence seq = ig.atomicSequence("seq", 0, true);
- return null;
- }
- }, IgniteException.class, "can not perform operation, because cluster inactive");
- }
-
- /**
- *
- */
- public void testFailGetLock() throws Exception {
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1C = backUpClient(0);
- Ignite ig2C = backUpClient(1);
-
- final Ignite ig3C = backUpClient(2);
-
- assertTrue(ig1P.active());
- assertTrue(ig2P.active());
- assertTrue(ig3P.active());
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
-
- stopPrimary(0);
-
- boolean exc = false;
-
- try {
- ig3C.active(true);
- }
- catch (IgniteException e) {
- exc = true;
-
- log.error("stack trace from remote node", e);
-
- for (Throwable t : e.getSuppressed())
- assertTrue(t.getMessage().contains("can't get lock during"));
- }
-
- if (!exc)
- fail();
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
- }
-
- /**
- *
- */
- public void testActivateAfterFailGetLock() throws Exception {
- Ignite ig1P = primary(0);
- Ignite ig2P = primary(1);
- Ignite ig3P = primary(2);
-
- Ignite ig1CP = primaryClient(0);
- Ignite ig2CP = primaryClient(1);
- Ignite ig3CP = primaryClient(2);
-
- Ignite ig1B = backUp(0);
- Ignite ig2B = backUp(1);
- Ignite ig3B = backUp(2);
-
- Ignite ig1CB = backUpClient(0);
- Ignite ig2CB = backUpClient(1);
- Ignite ig3CB = backUpClient(2);
-
- assertTrue(ig1P.active());
- assertTrue(ig2P.active());
- assertTrue(ig3P.active());
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1CB.active());
- assertTrue(!ig2CB.active());
- assertTrue(!ig3CB.active());
-
- stopPrimary(0);
-
- boolean exc = false;
-
- try {
- ig3CB.active(true);
- }
- catch (IgniteException e) {
- exc = true;
-
- log.error("stack trace from remote node", e);
-
- for (Throwable t : e.getSuppressed())
- assertTrue(t.getMessage().contains("can't get lock during"));
- }
-
- if (!exc)
- fail();
-
- assertTrue(!ig1B.active());
- assertTrue(!ig2B.active());
- assertTrue(!ig3B.active());
-
- assertTrue(!ig1CB.active());
- assertTrue(!ig2CB.active());
- assertTrue(!ig3CB.active());
-
- assertTrue(ig2P.active());
- assertTrue(ig3P.active());
-
- assertTrue(ig1CP.active());
- assertTrue(ig2CP.active());
- assertTrue(ig3CP.active());
-
- stopAllPrimary();
-
- ig2CB.active(true);
-
- assertTrue(ig1B.active());
- assertTrue(ig2B.active());
- assertTrue(ig3B.active());
-
- assertTrue(ig1CB.active());
- assertTrue(ig2CB.active());
- assertTrue(ig3CB.active());
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testDeActivateFromServerNode() throws Exception {
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig1.active(false);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testDeActivateFromClientNode() throws Exception {
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- Ignite ig1C = primaryClient(0);
- Ignite ig2C = primaryClient(1);
- Ignite ig3C = primaryClient(2);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig1C.active(false);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- assertTrue(!ig1C.active());
- assertTrue(!ig2C.active());
- assertTrue(!ig3C.active());
- }
-
- /**
- * @throws Exception if fail.
- */
- public void testDeActivateCheckCacheDestroy() throws Exception {
- String chName = "myCache";
-
- Ignite ig1 = primary(0);
- Ignite ig2 = primary(1);
- Ignite ig3 = primary(2);
-
- ig1.getOrCreateCache(chName);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig1.active(false);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- IgniteEx ex = (IgniteEx)ig1;
-
- GridCacheProcessor cache = ex.context().cache();
-
- assertTrue(F.isEmpty(cache.jcaches()));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteStandByClusterTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteStandByClusterTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteStandByClusterTest.java
deleted file mode 100644
index b248208..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/IgniteStandByClusterTest.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster;
-
-import java.util.Arrays;
-import java.util.Map;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.junit.Assert;
-
-/**
- *
- */
-public class IgniteStandByClusterTest extends GridCommonAbstractTest {
- private static final TcpDiscoveryIpFinder vmIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
-
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(vmIpFinder));
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
- cfg.setConsistentId(igniteInstanceName);
-
- return cfg;
- }
-
- public void testNotStartDynamicCachesOnClientAfterActivation() throws Exception {
- final String cacheName0 = "cache0";
- final String cacheName = "cache";
-
- IgniteConfiguration cfg1 = getConfiguration("serv1");
- IgniteConfiguration cfg2 = getConfiguration("serv2");
-
- IgniteConfiguration cfg3 = getConfiguration("client");
- cfg3.setCacheConfiguration(new CacheConfiguration(cacheName0));
-
- cfg3.setClientMode(true);
-
- IgniteEx ig1 = startGrid(cfg1);
- IgniteEx ig2 = startGrid(cfg2);
- IgniteEx ig3 = startGrid(cfg3);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- ig3.createCache(new CacheConfiguration<>(cacheName));
-
- assertNotNull(ig3.cache(cacheName));
- assertNotNull(ig1.cache(cacheName));
- assertNotNull(ig2.cache(cacheName));
-
- assertNotNull(ig1.cache(cacheName0));
- assertNotNull(ig3.cache(cacheName0));
- assertNotNull(ig2.cache(cacheName0));
-
- ig3.active(false);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- assertNotNull(ig1.cache(cacheName));
- assertNotNull(ig2.cache(cacheName));
-
- Map<String, GridCacheAdapter<?, ?>> caches = U.field(ig3.context().cache(), "caches");
-
- // Only system caches and cache0
- assertTrue(caches.size() == 3);
-
- assertNull(caches.get(cacheName));
-
- assertNotNull(caches.get(cacheName0));
-
- assertNotNull(ig3.cache(cacheName));
- }
-
- public void testStaticCacheStartAfterActivationWithCacheFilter() throws Exception {
- String cache1 = "cache1";
- String cache2 = "cache2";
- String cache3 = "cache3";
-
- IgniteConfiguration cfg1 = getConfiguration("node1");
-
- cfg1.setCacheConfiguration(
- new CacheConfiguration(cache1).setNodeFilter(new NodeFilterIgnoreByName("node2")));
-
- IgniteConfiguration cfg2 = getConfiguration("node2");
-
- cfg2.setCacheConfiguration(
- new CacheConfiguration(cache2).setNodeFilter(new NodeFilterIgnoreByName("node3")));
-
- IgniteConfiguration cfg3 = getConfiguration("node3");
-
- cfg3.setCacheConfiguration(
- new CacheConfiguration(cache3).setNodeFilter(new NodeFilterIgnoreByName("node1")));
-
- IgniteEx ig1 = startGrid(cfg1);
- IgniteEx ig2 = startGrid(cfg2);
- IgniteEx ig3 = startGrid(cfg3);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!ig3.active());
-
- for (IgniteEx ig: Arrays.asList(ig1,ig2,ig3)){
- Map<String, DynamicCacheDescriptor> desc = U.field(U.field(ig.context().cache(), "cachesInfo"), "registeredCaches");
-
- assertEquals(0, desc.size());
- }
-
- ig3.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(ig3.active());
-
- for (IgniteEx ig: Arrays.asList(ig1,ig2,ig3)){
- Map<String, DynamicCacheDescriptor> desc = U.field(
- U.field(ig.context().cache(), "cachesInfo"), "registeredCaches");
-
- assertEquals(5, desc.size());
-
- Map<String, GridCacheAdapter<?, ?>> caches = U.field(ig.context().cache(), "caches");
-
- assertEquals(4, caches.keySet().size());
- }
-
- Map<String, GridCacheAdapter<?, ?>> caches1 = U.field(ig1.context().cache(), "caches");
-
- Assert.assertNotNull(caches1.get(cache1));
- Assert.assertNotNull(caches1.get(cache2));
- Assert.assertNull(caches1.get(cache3));
-
- Map<String, GridCacheAdapter<?, ?>> caches2 = U.field(ig2.context().cache(), "caches");
-
- Assert.assertNull(caches2.get(cache1));
- Assert.assertNotNull(caches2.get(cache2));
- Assert.assertNotNull(caches2.get(cache3));
-
- Map<String, GridCacheAdapter<?, ?>> caches3 = U.field(ig3.context().cache(), "caches");
-
- Assert.assertNotNull(caches3.get(cache1));
- Assert.assertNull(caches3.get(cache2));
- Assert.assertNotNull(caches3.get(cache3));
- }
-
- private static class NodeFilterIgnoreByName implements IgnitePredicate<ClusterNode>{
- private final String name;
-
- private NodeFilterIgnoreByName(String name) {
- this.name = name;
- }
-
- @Override public boolean apply(ClusterNode node) {
- return !name.equals(node.consistentId());
- }
- }
-
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
- }
-
- @Override protected void afterTest() throws Exception {
- super.beforeTest();
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivateExtensionTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivateExtensionTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivateExtensionTest.java
deleted file mode 100644
index 1baf4a5..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivateExtensionTest.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.extended;
-
-import java.lang.reflect.Method;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.NearCacheConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.GridCacheAbstractFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.resource.GridSpringResourceContext;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-
-/**
- *
- */
-public class GridActivateExtensionTest extends GridCacheAbstractFullApiSelfTest {
- /** Condition id. */
- private int condId;
-
- /** Primary ip finder. */
- private TcpDiscoveryIpFinder primaryIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Back up ip finder. */
- private TcpDiscoveryIpFinder backUpIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Back up cluster. */
- private static Map<String, String> backUpCluster = new LinkedHashMap<>();
-
- /** Test name. */
- private final String testName = testName();
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setConsistentId("ConsId" + (condId++));
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(primaryIpFinder);
-
- PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
-
- pCfg.setPersistentStorePath(testName + "/db");
- pCfg.setWalArchivePath(testName + "/db/wal/archive");
- pCfg.setWalStorePath(testName + "/db/wal");
-
- cfg.setPersistentStoreConfiguration(pCfg);
-
- final MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setInitialSize(5 * 1024 * 1024);
- memPlcCfg.setMaxSize(5 * 1024 * 1024);
-
- memPlcCfg.setName("dfltMemPlc");
-
- memCfg.setMemoryPolicies(memPlcCfg);
-
- memCfg.setDefaultMemoryPolicyName(memPlcCfg.getName());
-
- memCfg.setPageSize(1024);
- memCfg.setConcurrencyLevel(64);
-
- cfg.setMemoryConfiguration(memCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName, true));
-
- super.beforeTestsStarted();
-
- if (onlyClient())
- return;
-
- int nodes = gridCount();
-
- if (nodes != condId)
- fail("checking fail");
-
- condId = 0;
-
- log.info("start backup cluster");
-
- for (String name: backUpCluster.keySet()){
- String n = name + "backUp";
-
- IgniteConfiguration cfg = getConfiguration(n);
- cfg.setActiveOnStart(false);
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(backUpIpFinder);
-
- startGrid(n, cfg);
-
- backUpCluster.put(name, n);
- }
-
- log.info("shutdown main cluster");
-
- for (String name : backUpCluster.keySet()){
- log.info(name + " stopping");
-
- stopGrid(name);
- }
-
- Ignite ig = grid(0);
-
- ig.active(true);
-
- log.info("activate backUp cluster");
-
- assertEquals(true, ig.active());
-
- Class c = getClass();
-
- if (isNearTest(c)) {
- for (int i = 0; i < gridCount(); i++) {
- if (ignite(i).configuration().isClientMode()) {
- Method m = getClientHasNearCacheMethod();
-
- Boolean r = (Boolean)m.invoke(this);
-
- if (r != null && r)
- ignite(i).createNearCache(null, new NearCacheConfiguration<>());
- else
- ignite(i).cache(null);
-
- break;
- }
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- stopAllGrids();
-
- backUpCluster.clear();
-
- condId = 0;
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), testName, true));
- }
-
- /** {@inheritDoc} */
- @Override protected Ignite startGrid(String gridName, GridSpringResourceContext ctx) throws Exception {
- if (!gridName.contains("backUp"))
- backUpCluster.put(gridName, gridName);
-
- return super.startGrid(gridName, ctx);
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteEx grid(String name) {
- String n = name;
-
- if (backUpCluster.containsKey(name))
- n = backUpCluster.get(name);
-
- return super.grid(n);
- }
-
- /**
- *
- */
- public boolean onlyClient(){
- for (int i = 0; i < gridCount(); i++)
- if (!grid(i).configuration().isClientMode())
- return false;
-
- return true;
- }
-
- /**
- * @param c class.
- */
- private boolean isNearTest(Class c) {
- Class p = c.getSuperclass();
-
- while (!p.equals(Object.class)) {
- if (p.equals(GridCacheNearOnlyMultiNodeFullApiSelfTest.class))
- return true;
-
- p = p.getSuperclass();
- }
-
- return false;
- }
-
- /**
- *
- */
- public Method getClientHasNearCacheMethod() throws NoSuchMethodException {
- Class p = getClass().getSuperclass();
-
- while (!p.equals(Object.class)) {
- if (p.equals(GridCacheNearOnlyMultiNodeFullApiSelfTest.class))
- return p.getDeclaredMethod("clientHasNearCache");
-
- p = p.getSuperclass();
- }
-
- return null;
- }
-
- /**
- *
- */
- protected String testName() {
- return getClass().getSimpleName();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationAtomicCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationAtomicCacheSuit.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationAtomicCacheSuit.java
deleted file mode 100644
index 397d8d0..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationAtomicCacheSuit.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.extended;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCacheAtomicNearEnabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeP2PDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicCopyOnReadDisabledMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicMultiNodeP2PDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearEnabledMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearOnlyMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicNearOnlyMultiNodeP2PDisabledFullApiSelfTest;
-
-/**
- *
- */
-public class GridActivationAtomicCacheSuit extends GridActivationCacheAbstractTestSuit {
- static {
- addTest(GridCacheAtomicClientOnlyMultiNodeFullApiSelfTest.class);
- addTest(GridCacheAtomicClientOnlyMultiNodeP2PDisabledFullApiSelfTest.class);
- addTest(GridCacheAtomicCopyOnReadDisabledMultiNodeFullApiSelfTest.class);
- addTest(GridCacheAtomicFullApiSelfTest.class);
- addTest(GridCacheAtomicMultiNodeFullApiSelfTest.class);
- addTest(GridCacheAtomicMultiNodeP2PDisabledFullApiSelfTest.class);
- addTest(GridCacheAtomicNearEnabledFullApiSelfTest.class);
- addTest(GridCacheAtomicNearEnabledMultiNodeFullApiSelfTest.class);
- addTest(GridCacheAtomicNearOnlyMultiNodeFullApiSelfTest.class);
- addTest(GridCacheAtomicNearOnlyMultiNodeP2PDisabledFullApiSelfTest.class);
-
-// addTest(GridCacheAtomicClientOnlyFairAffinityMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicClientOnlyMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicClientOnlyMultiJvmP2PDisabledFullApiSelfTest.class));
-// addTest(GridCacheAtomicCopyOnReadDisabledMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicFairAffinityMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicMultiJvmP2PDisabledFullApiSelfTest.class));
-// addTest(GridCacheAtomicOffHeapMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicOffHeapTieredMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicPrimaryWriteOrderFairAffinityMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicPrimaryWriteOrderMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicPrimaryWriteOrderMultiJvmP2PDisabledFullApiSelfTest.class));
-// addTest(GridCacheAtomicPrimaryWrityOrderOffHeapMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicPrimaryWrityOrderOffHeapTieredMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicNearEnabledFairAffinityMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicNearEnabledMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicNearEnabledPrimaryWriteOrderMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicNearOnlyMultiJvmFullApiSelfTest.class));
-// addTest(GridCacheAtomicNearOnlyMultiJvmP2PDisabledFullApiSelfTest.class));
- }
-
- /**
- *
- */
- public static TestSuite suite() {
- TestSuite suite = buildSuite();
-
- suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Atomic Cache");
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationCacheAbstractTestSuit.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationCacheAbstractTestSuit.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationCacheAbstractTestSuit.java
deleted file mode 100644
index 553546a..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationCacheAbstractTestSuit.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.extended;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import javassist.CannotCompileException;
-import javassist.ClassClassPath;
-import javassist.ClassPool;
-import javassist.CtClass;
-import javassist.NotFoundException;
-import junit.framework.TestSuite;
-import org.apache.ignite.IgniteException;
-
-/**
- *
- */
-public abstract class GridActivationCacheAbstractTestSuit {
- /** Tests. */
- private static final List<Class> tests = new ArrayList<>();
-
- /** Suffix. */
- private static final String SUFFIX = "Ex";
-
- /**
- * @return Suite.
- */
- protected static TestSuite buildSuite() {
- TestSuite suite = new TestSuite();
-
- for (Class c : tests)
- suite.addTestSuite(c);
-
- return suite;
- }
-
- /**
- * @param c Class.
- */
- protected static void addTest(Class c){
- tests.add(transform(c));
- }
-
-
- /**
- * @param c Class to transform.
- * @return Transformed class.
- */
- private static Class transform(Class c){
- try {
- ClassPool pool = ClassPool.getDefault();
-
- pool.insertClassPath(new ClassClassPath(GridActivationCacheAbstractTestSuit.class));
-
- String path = c.getProtectionDomain().getCodeSource().getLocation().getPath();
-
- CtClass ct = pool.get(c.getName());
-
- String name = c.getName() + SUFFIX;
-
- CtClass pr = pool.get(GridActivateExtensionTest.class.getName());
-
- pr.setName(name);
-
- pr.setSuperclass(ct);
-
- pr.writeFile(path);
-
- return Class.forName(name);
- }
- catch (IOException e) {
- System.out.println("Io exception: " + e.getMessage());
-
- throw new IgniteException(e);
- }
- catch (CannotCompileException e) {
- System.out.println("Cannot compile exception: " + e.getMessage());
-
- throw new IgniteException(e);
- }
- catch (NotFoundException e) {
- System.out.println("Not found exception: " + e.getMessage());
-
- throw new IgniteException(e);
- }
- catch (ClassNotFoundException e) {
- System.out.println("Class not found exception: " + e.getMessage());
-
- throw new IgniteException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java
deleted file mode 100644
index b6ef49c..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationLocalAndNearCacheSuit.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.extended;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.local.GridCacheLocalAtomicFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.local.GridCacheLocalFullApiSelfTest;
-
-/**
- *
- */
-public class GridActivationLocalAndNearCacheSuit extends GridActivationCacheAbstractTestSuit {
- static {
- addTest(GridCacheLocalAtomicFullApiSelfTest.class);
- addTest(GridCacheLocalFullApiSelfTest.class);
-
-// addTest(GridCacheNearOnlyFairAffinityMultiJvmFullApiSelfTest.class);
-// addTest(GridCacheNearOnlyMultiJvmFullApiSelfTest.class);
-// addTest(GridCacheNearOnlyMultiJvmP2PDisabledFullApiSelfTest.class);
- }
-
- /**
- *
- */
- public static TestSuite suite() {
- TestSuite suite = buildSuite();
-
- suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Local and Near Cache");
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationPartitionedCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationPartitionedCacheSuit.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationPartitionedCacheSuit.java
deleted file mode 100644
index fc437ad..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationPartitionedCacheSuit.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.extended;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledMultiNodeP2PDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedMultiNodeLongTxTimeoutFullApiTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheNearOnlyMultiNodeP2PDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedClientOnlyNoPrimaryFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedCopyOnReadDisabledMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest;
-
-/**
- *
- */
-public class GridActivationPartitionedCacheSuit extends GridActivationCacheAbstractTestSuit {
- static {
- addTest(CachePartitionedMultiNodeLongTxTimeoutFullApiTest.class);
- addTest(CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.class);
- addTest(GridCacheNearOnlyMultiNodeFullApiSelfTest.class);
- addTest(GridCacheNearOnlyMultiNodeP2PDisabledFullApiSelfTest.class);
- addTest(GridCachePartitionedClientOnlyNoPrimaryFullApiSelfTest.class);
- addTest(GridCachePartitionedCopyOnReadDisabledMultiNodeFullApiSelfTest.class);
- addTest(GridCachePartitionedFullApiSelfTest.class);
- addTest(GridCachePartitionedLateAffDisabledMultiNodeFullApiSelfTest.class);
- addTest(GridCachePartitionedMultiNodeFullApiSelfTest.class);
- addTest(GridCachePartitionedMultiNodeP2PDisabledFullApiSelfTest.class);
- addTest(GridCachePartitionedNearDisabledFullApiSelfTest.class);
- addTest(GridCachePartitionedNearDisabledMultiNodeFullApiSelfTest.class);
- addTest(GridCachePartitionedNearDisabledMultiNodeP2PDisabledFullApiSelfTest.class);
- addTest(GridCachePartitionedNearOnlyNoPrimaryFullApiSelfTest.class);
-
-// addTest(GridCachePartitionedMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedCopyOnReadDisabledMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedFairAffinityMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedMultiJvmP2PDisabledFullApiSelfTest.class);
-// addTest(GridCachePartitionedNearDisabledAtomicOffHeapTieredMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedNearDisabledFairAffinityMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedNearDisabledMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedNearDisabledMultiJvmP2PDisabledFullApiSelfTest.class);
-// addTest(GridCachePartitionedNearDisabledOffHeapMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedNearDisabledOffHeapTieredMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedOffHeapMultiJvmFullApiSelfTest.class);
-// addTest(GridCachePartitionedOffHeapTieredMultiJvmFullApiSelfTest.class);
- }
-
- /**
- *
- */
- public static TestSuite suite() {
- TestSuite suite = buildSuite();
-
- suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Partitioned Cache");
-
- return suite;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationReplicatedCacheSuit.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationReplicatedCacheSuit.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationReplicatedCacheSuit.java
deleted file mode 100644
index 6cdc87b..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/extended/GridActivationReplicatedCacheSuit.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.extended;
-
-import junit.framework.TestSuite;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.CacheReplicatedRendezvousAffinityExcludeNeighborsMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.CacheReplicatedRendezvousAffinityMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedAtomicMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedMultiNodeP2PDisabledFullApiSelfTest;
-import org.apache.ignite.internal.processors.cache.distributed.replicated.GridCacheReplicatedNearOnlyMultiNodeFullApiSelfTest;
-
-/**
- *
- */
-public class GridActivationReplicatedCacheSuit extends GridActivationCacheAbstractTestSuit {
- static {
- addTest(CacheReplicatedRendezvousAffinityExcludeNeighborsMultiNodeFullApiSelfTest.class);
- addTest(CacheReplicatedRendezvousAffinityMultiNodeFullApiSelfTest.class);
- addTest(GridCacheReplicatedAtomicFullApiSelfTest.class);
- addTest(GridCacheReplicatedFullApiSelfTest.class);
- addTest(GridCacheReplicatedMultiNodeFullApiSelfTest.class);
- addTest(GridCacheReplicatedAtomicMultiNodeFullApiSelfTest.class);
- addTest(GridCacheReplicatedMultiNodeP2PDisabledFullApiSelfTest.class);
- addTest(GridCacheReplicatedNearOnlyMultiNodeFullApiSelfTest.class);
-
-// tests.add(transform(GridCacheReplicatedAtomicMultiJvmFullApiSelfTest.class));
-// tests.add(transform(GridCacheReplicatedAtomicPrimaryWriteOrderMultiJvmFullApiSelfTest.class));
-// tests.add(transform(GridCacheReplicatedMultiJvmFullApiSelfTest.class));
-// tests.add(transform(GridCacheReplicatedMultiJvmP2PDisabledFullApiSelfTest.class));
-// tests.add(transform(GridCacheReplicatedNearOnlyMultiJvmFullApiSelfTest.class));
-// tests.add(transform(GridCacheReplicatedOffHeapMultiJvmFullApiSelfTest.class));
-// tests.add(transform(GridCacheReplicatedOffHeapTieredMultiJvmFullApiSelfTest.class));
- }
-
- /**
- *
- */
- public static TestSuite suite() {
- TestSuite suite = buildSuite();
-
- suite.setName("Activation Stand-by Cluster After Primary Cluster Stopped Check Replicated Cache");
-
- return suite;
- }
-}
[02/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToActiveCluster.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToActiveCluster.java
deleted file mode 100644
index 6042267..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToActiveCluster.java
+++ /dev/null
@@ -1,431 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join;
-
-import java.util.Map;
-import org.apache.ignite.cache.database.standbycluster.AbstractNodeJoinTemplate;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.junit.Assert;
-
-/**
- *
- */
-public class JoinActiveNodeToActiveCluster extends AbstractNodeJoinTemplate {
- /**
- *
- */
- @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(true),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheOnlySystem()
- ).nodeConfiguration(
- cfg(name(3)).setActiveOnStart(true)
- ).afterNodeJoin(
- b.checkCacheOnlySystem()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheOnlySystem()
- );
-
- return b;
- }
-
- /**
- *
- */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(true),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheOnlySystem()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /**
- *
- */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheNotEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /**
- *
- */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheNotEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /**
- *
- */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(true)
- .setCacheConfiguration(atomicCfg()),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(3, desc.size());
-
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(3, caches.size());
- }
- }
- }
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- .setCacheConfiguration(transactionCfg())
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- // Server node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinWithOutConfiguration() throws Exception {
- withOutConfigurationTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
- staticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testStaticCacheConfigurationInCluster() throws Exception {
- staticCacheConfigurationInClusterTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
- staticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
- staticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- // Client node join.
-
- /**
- *
- */
- @Override public void testJoinClientWithOutConfiguration() throws Exception {
- joinClientWithOutConfigurationTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
- joinClientStaticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
- joinClientStaticCacheConfigurationInClusterTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
- joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /**
- *
- */
- @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
- joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
- return withOutConfigurationTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
- return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
- return staticCacheConfigurationInClusterTemplate()
- .nodeConfiguration(setClient)
- .afterActivate(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient()) {
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
- else {
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
- }
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- }
- }).afterNodeJoin(
- new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient()) {
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
- else {
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
- }
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- }
- }).setEnd(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient()) {
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
- else {
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
- }
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- }
- });
- }
-
- @Override
- public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return staticCacheConfigurationDifferentOnBothTemplate()
- .nodeConfiguration(setClient)
- .afterActivate(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient()) {
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
- else {
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- }
- })
- .afterNodeJoin(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 4; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertNotNull(ig.context().cache().cache(cache1));
-
- Assert.assertNotNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertEquals(4, caches.size());
- else
- Assert.assertEquals(3, caches.size());
- }
- }
- })
- .setEnd(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 4; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertNotNull(ig.context().cache().cache(cache1));
-
- Assert.assertNotNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertEquals(4, caches.size());
- else
- Assert.assertEquals(3, caches.size());
- }
- }
- });
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
- return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToInActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToInActiveCluster.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToInActiveCluster.java
deleted file mode 100644
index 62ab114..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinActiveNodeToInActiveCluster.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join;
-
-import org.apache.ignite.cache.database.standbycluster.AbstractNodeJoinTemplate;
-
-/**
- *
- */
-public class JoinActiveNodeToInActiveCluster extends AbstractNodeJoinTemplate {
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(false),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheOnlySystem()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(false),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3)).setActiveOnStart(true)
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(false)
- .setCacheConfiguration(atomicCfg()),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(true)
- .setCacheConfiguration(transactionCfg())
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- // Server node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinWithOutConfiguration() throws Exception {
- withOutConfigurationTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
- staticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationInCluster() throws Exception {
- staticCacheConfigurationInClusterTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
- staticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
- staticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- // Client node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinClientWithOutConfiguration() throws Exception {
- joinClientWithOutConfigurationTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
- joinClientStaticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
- joinClientStaticCacheConfigurationInClusterTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
- joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
- joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
- return withOutConfigurationTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
- return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
- return staticCacheConfigurationInClusterTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return staticCacheConfigurationDifferentOnBothTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
- return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToActiveCluster.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToActiveCluster.java
deleted file mode 100644
index d0cc7b3..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToActiveCluster.java
+++ /dev/null
@@ -1,356 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join;
-
-import java.util.Map;
-import org.apache.ignite.cache.database.standbycluster.AbstractNodeJoinTemplate;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
-import org.junit.Assert;
-
-/**
- *
- */
-public class JoinInActiveNodeToActiveCluster extends AbstractNodeJoinTemplate {
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(true),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheOnlySystem()
- ).nodeConfiguration(
- cfg(name(3)).setActiveOnStart(false)
- ).afterNodeJoin(
- b.checkCacheOnlySystem()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheOnlySystem()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(true),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheOnlySystem()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheNotEmpty()
- ).nodeConfiguration(
- cfg(name(3)).setActiveOnStart(false)
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(true)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- b.checkCacheNotEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(true)
- .setCacheConfiguration(atomicCfg()),
- cfg(name(1)).setActiveOnStart(true),
- cfg(name(2)).setActiveOnStart(true)
- ).afterClusterStarted(
- new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(3, desc.size());
-
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(3, caches.size());
- }
- }
- }
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(false)
- .setCacheConfiguration(transactionCfg())
- ).afterNodeJoin(
- b.checkCacheNotEmpty()
- ).stateAfterJoin(
- true
- ).afterDeActivate(
- b.checkCacheEmpty()
- ).setEnd(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- // Server node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinWithOutConfiguration() throws Exception {
- withOutConfigurationTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
- staticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationInCluster() throws Exception {
- staticCacheConfigurationInClusterTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
- staticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
- staticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- // Client node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinClientWithOutConfiguration() throws Exception {
- joinClientWithOutConfigurationTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
- joinClientStaticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
- joinClientStaticCacheConfigurationInClusterTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
- joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
- joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
- return withOutConfigurationTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
- return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
- return staticCacheConfigurationInClusterTemplate()
- .nodeConfiguration(setClient)
- .afterNodeJoin(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient()) {
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
- else {
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
- }
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- }
- }).setEnd(new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 3; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient()) {
- Assert.assertNotNull(ig.context().cache().cache(cache1));
- Assert.assertNotNull(ig.context().cache().cache(cache2));
- }
- else {
- Assert.assertNull(ig.context().cache().cache(cache1));
- Assert.assertNull(ig.context().cache().cache(cache2));
- }
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- Assert.assertEquals(4, caches.size());
- }
- }
- });
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
- return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return staticCacheConfigurationDifferentOnBothTemplate()
- .nodeConfiguration(setClient)
- .afterNodeJoin(
- new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 4; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertNotNull(ig.context().cache().cache(cache1));
-
- Assert.assertNotNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertEquals(4, caches.size());
- else
- Assert.assertEquals(3, caches.size());
- }
- }
- }
- ).setEnd(
- new Runnable() {
- @Override public void run() {
- for (int i = 0; i < 4; i++) {
- IgniteEx ig = grid(name(i));
-
- Map<String, DynamicCacheDescriptor> desc = cacheDescriptors(ig);
-
- Assert.assertEquals(4, desc.size());
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertNotNull(ig.context().cache().cache(cache1));
-
- Assert.assertNotNull(ig.context().cache().cache(cache2));
-
- Map<String, GridCacheAdapter> caches = caches(ig);
-
- if (!ig.context().discovery().localNode().isClient())
- Assert.assertEquals(4, caches.size());
- else
- Assert.assertEquals(3, caches.size());
- }
- }
- }
- );
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToInActiveCluster.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToInActiveCluster.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToInActiveCluster.java
deleted file mode 100644
index 7333c2e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/JoinInActiveNodeToInActiveCluster.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join;
-
-import org.apache.ignite.cache.database.standbycluster.AbstractNodeJoinTemplate;
-
-/**
- *
- */
-public class JoinInActiveNodeToInActiveCluster extends AbstractNodeJoinTemplate {
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(false),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3)).setActiveOnStart(false)
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheOnlySystem()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0)).setActiveOnStart(false),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3)).setActiveOnStart(false)
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations()),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(false)
- .setCacheConfiguration(allCacheConfigurations())
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- /** {@inheritDoc} */
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = builder();
-
- b.clusterConfiguration(
- cfg(name(0))
- .setActiveOnStart(false)
- .setCacheConfiguration(transactionCfg()),
- cfg(name(1)).setActiveOnStart(false),
- cfg(name(2)).setActiveOnStart(false)
- ).afterClusterStarted(
- b.checkCacheEmpty()
- ).nodeConfiguration(
- cfg(name(3))
- .setActiveOnStart(false)
- .setCacheConfiguration(atomicCfg())
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- // Server node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinWithOutConfiguration() throws Exception {
- withOutConfigurationTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationOnJoin() throws Exception {
- staticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationInCluster() throws Exception {
- staticCacheConfigurationInClusterTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationSameOnBoth() throws Exception {
- staticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testStaticCacheConfigurationDifferentOnBoth() throws Exception {
- staticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- // Client node join.
-
- /** {@inheritDoc} */
- @Override public void testJoinClientWithOutConfiguration() throws Exception {
- joinClientWithOutConfigurationTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationOnJoin() throws Exception {
- joinClientStaticCacheConfigurationOnJoinTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationInCluster() throws Exception {
- joinClientStaticCacheConfigurationInClusterTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationSameOnBoth() throws Exception {
- joinClientStaticCacheConfigurationSameOnBothTemplate().execute();
- }
-
- /** {@inheritDoc} */
- @Override public void testJoinClientStaticCacheConfigurationDifferentOnBoth() throws Exception {
- joinClientStaticCacheConfigurationDifferentOnBothTemplate().execute();
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
- return withOutConfigurationTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
- return staticCacheConfigurationOnJoinTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
- return staticCacheConfigurationInClusterTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationSameOnBothTemplate() throws Exception {
- return staticCacheConfigurationSameOnBothTemplate().nodeConfiguration(setClient);
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return staticCacheConfigurationDifferentOnBothTemplate().nodeConfiguration(setClient);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java
deleted file mode 100644
index fc4b5ce..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToActiveClusterWithPersistence.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join.persistence;
-
-import org.apache.ignite.cache.database.standbycluster.join.JoinActiveNodeToActiveCluster;
-import org.apache.ignite.configuration.IgniteConfiguration;
-
-/**
- *
- */
-public class JoinActiveNodeToActiveClusterWithPersistence extends JoinActiveNodeToActiveCluster {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration cfg(String name) throws Exception {
- return persistentCfg(super.cfg(name));
- }
-
- private JoinNodeTestPlanBuilder persistent(JoinNodeTestPlanBuilder b) {
- b.afterClusterStarted(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = persistent(super.withOutConfigurationTemplate());
-
- b.afterActivate(b.checkCacheOnlySystem());
-
- return b;
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = persistent(super.joinClientWithOutConfigurationTemplate());
-
- b.afterActivate(b.checkCacheOnlySystem());
-
- return b;
- }
-
- @Override public void testJoinWithOutConfiguration() throws Exception {
- withOutConfigurationTemplate().execute();
- }
-
- @Override public void testJoinClientWithOutConfiguration() throws Exception {
- joinClientWithOutConfigurationTemplate().execute();
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationOnJoinTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationInClusterTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationSameOnBothTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationDifferentOnBothTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationOnJoinTemplate() throws Exception {
- return persistent(super.joinClientStaticCacheConfigurationOnJoinTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception {
- return persistent(super.joinClientStaticCacheConfigurationInClusterTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return persistent(super.joinClientStaticCacheConfigurationDifferentOnBothTemplate());
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java
deleted file mode 100644
index 9981322..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinActiveNodeToInActiveClusterWithPersistence.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join.persistence;
-
-import org.apache.ignite.cache.database.standbycluster.join.JoinActiveNodeToInActiveCluster;
-import org.apache.ignite.configuration.IgniteConfiguration;
-
-/**
- *
- */
-public class JoinActiveNodeToInActiveClusterWithPersistence extends JoinActiveNodeToInActiveCluster {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration cfg(String name) throws Exception {
- return persistentCfg(super.cfg(name));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java
deleted file mode 100644
index 4731647..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToActiveClusterWithPersistence.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join.persistence;
-
-import org.apache.ignite.cache.database.standbycluster.join.JoinInActiveNodeToActiveCluster;
-import org.apache.ignite.configuration.IgniteConfiguration;
-
-/**
- *
- */
-public class JoinInActiveNodeToActiveClusterWithPersistence extends JoinInActiveNodeToActiveCluster {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration cfg(String name) throws Exception {
- return persistentCfg(super.cfg(name));
- }
-
- private JoinNodeTestPlanBuilder persistent(JoinNodeTestPlanBuilder b) {
- b.afterClusterStarted(
- b.checkCacheEmpty()
- ).stateAfterJoin(
- false
- ).afterNodeJoin(
- b.checkCacheEmpty()
- ).afterActivate(
- b.checkCacheNotEmpty()
- );
-
- return b;
- }
-
- @Override public JoinNodeTestPlanBuilder withOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = persistent(super.withOutConfigurationTemplate());
-
- b.afterActivate(b.checkCacheOnlySystem());
-
- return b;
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientWithOutConfigurationTemplate() throws Exception {
- JoinNodeTestPlanBuilder b = persistent(super.joinClientWithOutConfigurationTemplate());
-
- b.afterActivate(b.checkCacheOnlySystem());
-
- return b;
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationInClusterTemplate() throws Exception{
- return persistent(super.joinClientStaticCacheConfigurationInClusterTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder joinClientStaticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return persistent(super.joinClientStaticCacheConfigurationDifferentOnBothTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationOnJoinTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationOnJoinTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationInClusterTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationInClusterTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationSameOnBothTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationSameOnBothTemplate());
- }
-
- @Override public JoinNodeTestPlanBuilder staticCacheConfigurationDifferentOnBothTemplate() throws Exception {
- return persistent(super.staticCacheConfigurationDifferentOnBothTemplate());
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java
deleted file mode 100644
index 01154b1..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/join/persistence/JoinInActiveNodeToInActiveClusterWithPersistence.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.join.persistence;
-
-import org.apache.ignite.cache.database.standbycluster.join.JoinInActiveNodeToInActiveCluster;
-import org.apache.ignite.configuration.IgniteConfiguration;
-
-/**
- *
- */
-public class JoinInActiveNodeToInActiveClusterWithPersistence extends JoinInActiveNodeToInActiveCluster {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration cfg(String name) throws Exception {
- return persistentCfg(super.cfg(name));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
deleted file mode 100644
index e5f34f8..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteAbstractStandByClientReconnectTest.java
+++ /dev/null
@@ -1,336 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.reconnect;
-
-import com.google.common.collect.Sets;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import org.apache.ignite.cluster.ClusterNode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.events.Event;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgnitePredicate;
-import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage;
-import org.apache.ignite.spi.discovery.DiscoverySpiListener;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-import org.jetbrains.annotations.Nullable;
-import org.junit.Assert;
-
-import static org.apache.ignite.events.EventType.EVT_CLIENT_NODE_DISCONNECTED;
-
-/**
- *
- */
-public abstract class IgniteAbstractStandByClientReconnectTest extends GridCommonAbstractTest {
-
- private static final TcpDiscoveryVmIpFinder vmIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- private static final TcpDiscoveryVmIpFinder clientIpFinder = new TcpDiscoveryVmIpFinder(true);
-
- protected final String node1 = "node1";
- protected final String node2 = "node2";
- protected final String nodeClient = "nodeClient";
-
- protected final String ccfg1staticName = "cache1static";
- protected final String ccfg2staticName = "cache2static";
- protected final String ccfg3staticName = "cache3static";
-
- protected final String ccfg1staticWithFilterName = "ccfg1staticWithFilter";
- protected final String ccfg2staticWithFilterName = "ccfg2staticWithFilter";
- protected final String ccfg3staticWithFilterName = "ccfg3staticWithFilter";
-
- protected final String ccfgDynamicName = "ccfgDynamic";
- protected final String ccfgDynamicWithFilterName = "ccfgDynamicWithFilter";
-
- protected final CacheConfiguration ccfg1static = new CacheConfiguration(ccfg1staticName);
- protected final CacheConfiguration ccfg2static = new CacheConfiguration(ccfg2staticName);
- protected final CacheConfiguration ccfg3static = new CacheConfiguration(ccfg3staticName);
-
- protected final CacheConfiguration ccfg1staticWithFilter =
- new CacheConfiguration(ccfg1staticWithFilterName).setNodeFilter(new FilterNode(node2));
-
- protected final CacheConfiguration ccfg2staticWithFilter =
- new CacheConfiguration(ccfg2staticWithFilterName).setNodeFilter(new FilterNode(nodeClient));
-
- protected final CacheConfiguration ccfg3staticWithFilter =
- new CacheConfiguration(ccfg3staticWithFilterName).setNodeFilter(new FilterNode(node1));
-
- protected final CacheConfiguration ccfgDynamic = new CacheConfiguration(ccfgDynamicName);
-
- protected final CacheConfiguration ccfgDynamicWithFilter =
- new CacheConfiguration(ccfgDynamicWithFilterName).setNodeFilter(new FilterNode(node2));
-
- protected final Set<String> staticCacheNames = Sets.newHashSet(
- ccfg1staticName, ccfg2staticName, ccfg3staticName,
- ccfg1staticWithFilterName, ccfg2staticWithFilterName, ccfg3staticWithFilterName
- );
-
- protected final Set<String> allCacheNames = Sets.newHashSet(
- ccfg1staticName, ccfg2staticName, ccfg3staticName,
- ccfg1staticWithFilterName, ccfg2staticWithFilterName, ccfg3staticWithFilterName,
- ccfgDynamicName, ccfgDynamicWithFilterName
- );
-
- @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(name);
-
- if (!nodeClient.equals(name))
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(vmIpFinder));
- else {
- clientIpFinder.setAddresses(Collections.singletonList("127.0.0.1:47501"));
-
- cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(clientIpFinder));
- }
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
- cfg.setConsistentId(name);
-
- return cfg;
- }
-
- protected void addDisconnectListener(
- final CountDownLatch disconnectedLatch,
- final CountDownLatch reconnectedLatch
- ) {
- grid(nodeClient).events().localListen(new IgnitePredicate<Event>() {
- @Override public boolean apply(Event event) {
- switch (event.type()) {
- case EventType.EVT_CLIENT_NODE_DISCONNECTED:
- info("Client disconnected");
-
- disconnectedLatch.countDown();
-
- break;
- case EventType.EVT_CLIENT_NODE_RECONNECTED:
- info("Client reconnected");
-
- reconnectedLatch.countDown();
- }
-
- return true;
- }
- }, EventType.EVT_CLIENT_NODE_DISCONNECTED, EventType.EVT_CLIENT_NODE_RECONNECTED);
- }
-
- protected void checkDescriptors(IgniteEx ig, Set<String> cacheNames) {
- Collection<DynamicCacheDescriptor> descs = ig.context().cache().cacheDescriptors().values();
-
- assertEquals("Node name: " + ig.name(), cacheNames.size() + 2, descs.size());
-
- int systemCnt = 0;
-
- for (DynamicCacheDescriptor desc : descs)
- if (!CU.isSystemCache(desc.cacheName()))
- assertTrue(desc.cacheName(), cacheNames.contains(desc.cacheName()));
- else
- systemCnt++;
-
- assertEquals(2, systemCnt);
- }
-
- protected void startNodes(CountDownLatch activateLatch) throws Exception {
- IgniteConfiguration cfg1 = getConfiguration(node1)
- .setCacheConfiguration(ccfg1static, ccfg1staticWithFilter);
-
- IgniteConfiguration cfg2 = getConfiguration(node2)
- .setCacheConfiguration(ccfg2static, ccfg2staticWithFilter);
-
- IgniteConfiguration cfg3 = getConfiguration(nodeClient)
- .setCacheConfiguration(ccfg3static, ccfg3staticWithFilter);
-
- if (activateLatch != null)
- cfg3.setDiscoverySpi(
- new AwaitTcpDiscoverySpi(activateLatch)
- .setIpFinder(clientIpFinder)
- );
-
- cfg3.setClientMode(true);
-
- IgniteEx ig1 = startGrid(cfg1);
- IgniteEx ig2 = startGrid(cfg2);
- IgniteEx client = startGrid(cfg3);
- }
-
- protected void checkStaticCaches() {
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- Assert.assertNotNull(ig1.cache(ccfg1staticName));
- Assert.assertNotNull(ig1.cache(ccfg2staticName));
- Assert.assertNotNull(ig1.cache(ccfg3staticName));
-
- Assert.assertNotNull(ig1.cache(ccfg1staticWithFilterName));
- Assert.assertNotNull(ig1.cache(ccfg2staticWithFilterName));
-
- Assert.assertNotNull(ig2.cache(ccfg1staticName));
- Assert.assertNotNull(ig2.cache(ccfg2staticName));
- Assert.assertNotNull(ig2.cache(ccfg3staticName));
-
- Assert.assertNotNull(ig2.cache(ccfg3staticWithFilterName));
- Assert.assertNotNull(ig2.cache(ccfg2staticWithFilterName));
-
- Assert.assertNotNull(client.cache(ccfg1staticName));
- Assert.assertNotNull(client.cache(ccfg2staticName));
- Assert.assertNotNull(client.cache(ccfg3staticName));
-
- Assert.assertNotNull(client.cache(ccfg3staticWithFilterName));
- Assert.assertNotNull(client.cache(ccfg1staticWithFilterName));
- }
-
- protected void checkAllCaches() {
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- checkStaticCaches();
-
- Assert.assertNotNull(ig1.cache(ccfgDynamicName));
- Assert.assertNotNull(ig1.cache(ccfgDynamicWithFilterName));
-
- Assert.assertNotNull(ig2.cache(ccfgDynamicName));
-
- Assert.assertNotNull(client.cache(ccfgDynamicName));
- Assert.assertNotNull(client.cache(ccfgDynamicWithFilterName));
- }
-
- protected void checkOnlySystemCaches() {
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- Assert.assertNull(ig1.cache(ccfg1staticName));
- Assert.assertNull(ig1.cache(ccfg2staticName));
- Assert.assertNull(ig1.cache(ccfg3staticName));
-
- Assert.assertNull(ig1.cache(ccfg1staticWithFilterName));
- Assert.assertNull(ig1.cache(ccfg2staticWithFilterName));
-
- Assert.assertNull(ig2.cache(ccfg1staticName));
- Assert.assertNull(ig2.cache(ccfg2staticName));
- Assert.assertNull(ig2.cache(ccfg3staticName));
-
- Assert.assertNull(ig2.cache(ccfg3staticWithFilterName));
- Assert.assertNull(ig2.cache(ccfg2staticWithFilterName));
-
- Assert.assertNull(client.cache(ccfg1staticName));
- Assert.assertNull(client.cache(ccfg2staticName));
- Assert.assertNull(client.cache(ccfg3staticName));
-
- Assert.assertNull(client.cache(ccfg3staticWithFilterName));
- Assert.assertNull(client.cache(ccfg1staticWithFilterName));
-
- checkDescriptors(ig1,Collections.<String>emptySet());
- checkDescriptors(ig2,Collections.<String>emptySet());
- checkDescriptors(client, Collections.<String>emptySet());
- }
-
- private static class FilterNode implements IgnitePredicate<ClusterNode> {
-
- private final String consistentId;
-
- private FilterNode(String id) {
- consistentId = id;
- }
-
- @Override public boolean apply(ClusterNode node) {
- return !consistentId.equals(node.consistentId());
- }
- }
-
- private static class AwaitTcpDiscoverySpi extends TcpDiscoverySpi {
-
- private final CountDownLatch latch;
-
- private AwaitTcpDiscoverySpi(CountDownLatch latch) {
- this.latch = latch;
- }
-
- @Override public void setListener(@Nullable DiscoverySpiListener lsnr) {
- super.setListener(new AwaitDiscoverySpiListener(latch, lsnr));
- }
- }
-
- private static class AwaitDiscoverySpiListener implements DiscoverySpiListener {
-
- private final CountDownLatch latch;
-
- private final DiscoverySpiListener delegate;
-
- private AwaitDiscoverySpiListener(
- CountDownLatch latch,
- DiscoverySpiListener delegate
- ) {
- this.latch = latch;
- this.delegate = delegate;
- }
-
- @Override public void onLocalNodeInitialized(ClusterNode locNode) {
- delegate.onLocalNodeInitialized(locNode);
- }
-
- @Override public void onDiscovery(
- int type,
- long topVer,
- ClusterNode node,
- Collection<ClusterNode> topSnapshot,
- @Nullable Map<Long, Collection<ClusterNode>> topHist,
- @Nullable DiscoverySpiCustomMessage data
- ) {
- delegate.onDiscovery(type, topVer, node, topSnapshot, topHist, data);
-
- if (type == EVT_CLIENT_NODE_DISCONNECTED)
- try {
- System.out.println("Await cluster change state");
-
- latch.await();
- }
- catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
- }
-
- @Override protected void afterTest() throws Exception {
- super.beforeTest();
-
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", true));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectTest.java
deleted file mode 100644
index 1266f89..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/standbycluster/reconnect/IgniteStandByClientReconnectTest.java
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database.standbycluster.reconnect;
-
-import java.util.concurrent.CountDownLatch;
-import org.apache.ignite.internal.IgniteEx;
-
-/**
- *
- */
-public class IgniteStandByClientReconnectTest extends IgniteAbstractStandByClientReconnectTest {
-
- public void testActiveClientReconnectToActiveCluster() throws Exception {
- CountDownLatch activateLatch = new CountDownLatch(1);
-
- startNodes(activateLatch);
-
- info(">>>> star grid");
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- client.active(true);
-
- info(">>>> activate grid");
-
- checkDescriptors(ig1, staticCacheNames);
- checkDescriptors(ig2, staticCacheNames);
- checkDescriptors(client, staticCacheNames);
-
- checkStaticCaches();
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- info(">>>> stop servers");
-
- stopGrid(node2);
-
- disconnectedLatch.await();
-
- ig2 = startGrid(getConfiguration(node2));
-
- info(">>>> activate new servers");
-
- ig1.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
-
- activateLatch.countDown();
-
- info(">>>> reconnect client");
-
- reconnectedLatch.await();
-
- info(">>>> client reconnected");
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- checkAllCaches();
- }
-
- public void testActiveClientReconnectToInActiveCluster() throws Exception {
- CountDownLatch activateLatch = new CountDownLatch(1);
-
- startNodes(activateLatch);
-
- info(">>>> star grid");
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- info(">>>> activate grid");
-
- client.active(true);
-
- checkStaticCaches();
-
- checkDescriptors(ig1, staticCacheNames);
- checkDescriptors(ig2, staticCacheNames);
- checkDescriptors(client, staticCacheNames);
-
- info(">>>> dynamic start [" + ccfgDynamicName + ", " + ccfgDynamicWithFilterName + "]");
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- info(">>>> stop " + node2);
-
- stopGrid(node2);
-
- disconnectedLatch.await();
-
- ig1.active(false);
-
- activateLatch.countDown();
-
- info(">>>> restart " + node2);
-
- ig2 = startGrid(getConfiguration(node2));
-
- reconnectedLatch.await();
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- client.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- checkAllCaches();
- }
-
- public void testInActiveClientReconnectToActiveCluster() throws Exception {
- CountDownLatch activateLatch = new CountDownLatch(1);
-
- startNodes(activateLatch);
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- stopGrid(node2);
-
- disconnectedLatch.await();
-
- ig2 = startGrid(getConfiguration(node2));
-
- ig1.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
-
- checkDescriptors(ig1, staticCacheNames);
- checkDescriptors(ig2, staticCacheNames);
-
- activateLatch.countDown();
-
- reconnectedLatch.await();
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkDescriptors(ig1, staticCacheNames);
- checkDescriptors(ig2, staticCacheNames);
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- checkAllCaches();
- }
-
- public void testInActiveClientReconnectToInActiveCluster() throws Exception {
- startNodes(null);
-
- IgniteEx ig1 = grid(node1);
- IgniteEx ig2 = grid(node2);
- IgniteEx client = grid(nodeClient);
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- final CountDownLatch disconnectedLatch = new CountDownLatch(1);
- final CountDownLatch reconnectedLatch = new CountDownLatch(1);
-
- addDisconnectListener(disconnectedLatch, reconnectedLatch);
-
- stopGrid(node2);
-
- disconnectedLatch.await();
-
- ig2 = startGrid(getConfiguration(node2));
-
- reconnectedLatch.await();
-
- assertTrue(!ig1.active());
- assertTrue(!ig2.active());
- assertTrue(!client.active());
-
- client.active(true);
-
- assertTrue(ig1.active());
- assertTrue(ig2.active());
- assertTrue(client.active());
-
- checkStaticCaches();
-
- client.createCache(ccfgDynamic);
-
- client.createCache(ccfgDynamicWithFilter);
-
- checkDescriptors(ig1, allCacheNames);
- checkDescriptors(ig2, allCacheNames);
- checkDescriptors(client, allCacheNames);
-
- checkAllCaches();
- }
-}
[08/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsContinuousRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsContinuousRestartTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsContinuousRestartTest.java
deleted file mode 100644
index a149836..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsContinuousRestartTest.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.util.Map;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
- /** */
- private static final int GRID_CNT = 4;
-
- /** */
- private static final int ENTRIES_COUNT = 10_000;
-
- /** */
- public static final String CACHE_NAME = "cache1";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setMaxSize(400 * 1024 * 1024);
- memPlcCfg.setInitialSize(400 * 1024 * 1024);
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(memCfg);
-
- CacheConfiguration ccfg1 = new CacheConfiguration();
-
- ccfg1.setName(CACHE_NAME);
- ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg1.setAffinity(new RendezvousAffinityFunction(false, 128));
- ccfg1.setBackups(2);
-
- cfg.setCacheConfiguration(ccfg1);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_1000_500_1_1() throws Exception {
- checkRebalancingDuringLoad(1000, 500, 1, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_8000_500_1_1() throws Exception {
- checkRebalancingDuringLoad(8000, 500, 1, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_1000_20000_1_1() throws Exception {
- checkRebalancingDuringLoad(1000, 20000, 1, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_8000_8000_1_1() throws Exception {
- checkRebalancingDuringLoad(8000, 8000, 1, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_1000_500_8_1() throws Exception {
- checkRebalancingDuringLoad(1000, 500, 8, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_8000_500_8_1() throws Exception {
- checkRebalancingDuringLoad(8000, 500, 8, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_1000_20000_8_1() throws Exception {
- checkRebalancingDuringLoad(1000, 20000, 8, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_8000_8000_8_1() throws Exception {
- checkRebalancingDuringLoad(8000, 8000, 8, 1);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_1000_500_8_16() throws Exception {
- checkRebalancingDuringLoad(1000, 500, 8, 16);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_8000_500_8_16() throws Exception {
- checkRebalancingDuringLoad(8000, 500, 8, 16);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_1000_20000_8_16() throws Exception {
- checkRebalancingDuringLoad(1000, 20000, 8, 16);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRebalancingDuringLoad_8000_8000_8_16() throws Exception {
- checkRebalancingDuringLoad(8000, 8000, 8, 16);
- }
-
- /**
- * @throws Exception if failed.
- */
- private void checkRebalancingDuringLoad(
- int restartDelay,
- int checkpointDelay,
- int threads,
- final int batch
- ) throws Exception {
-
- startGrids(GRID_CNT);
-
- final Ignite load = ignite(0);
-
- try (IgniteDataStreamer<Object, Object> s = load.dataStreamer(CACHE_NAME)) {
- s.allowOverwrite(true);
-
- for (int i = 0; i < ENTRIES_COUNT; i++)
- s.addData(i, i);
- }
-
- final AtomicBoolean done = new AtomicBoolean(false);
-
- IgniteInternalFuture<?> busyFut = GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
- /** {@inheritDoc} */
- @Override public Object call() throws Exception {
- IgniteCache<Object, Object> cache = load.cache(CACHE_NAME);
- Random rnd = ThreadLocalRandom.current();
-
- while (!done.get()) {
- Map<Integer, Integer> map = new TreeMap<>();
-
- for (int i = 0; i < batch; i++)
- map.put(rnd.nextInt(ENTRIES_COUNT), rnd.nextInt());
-
- cache.putAll(map);
- }
-
- return null;
- }
- }, threads, "updater");
-
- long end = System.currentTimeMillis() + 90_000;
-
- Random rnd = ThreadLocalRandom.current();
-
- while (System.currentTimeMillis() < end) {
- int idx = rnd.nextInt(GRID_CNT - 1) + 1;
-
- stopGrid(idx);
-
- U.sleep(restartDelay);
-
- startGrid(idx);
-
- U.sleep(restartDelay);
- }
-
- done.set(true);
-
- busyFut.get();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsDynamicCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsDynamicCacheTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsDynamicCacheTest.java
deleted file mode 100644
index 63bf557..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsDynamicCacheTest.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class IgnitePdsDynamicCacheTest extends IgniteDbDynamicCacheSelfTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(200 * 1024 * 1024);
- memPlcCfg.setMaxSize(200 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- if ("client".equals(gridName))
- cfg.setClientMode(true);
-
- return cfg;
- }
-
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- System.setProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC, "true");
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- super.beforeTestsStarted();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- System.clearProperty(GridCacheDatabaseSharedManager.IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
-
- deleteWorkFiles();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testRestartAndCreate() throws Exception {
- startGrids(3);
-
- Ignite ignite = ignite(0);
-
- CacheConfiguration ccfg1 = new CacheConfiguration();
-
- ccfg1.setName("cache1");
- ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg1.setRebalanceMode(CacheRebalanceMode.NONE);
- ccfg1.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- CacheConfiguration ccfg2 = new CacheConfiguration();
-
- ccfg2.setName("cache2");
- ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- ccfg2.setRebalanceMode(CacheRebalanceMode.NONE);
- ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- ignite.createCache(ccfg1);
- ignite.createCache(ccfg2);
-
- int iterations = 20;
-
- long stopTime = U.currentTimeMillis() + 20_000;
-
- for (int k = 0; k < iterations && U.currentTimeMillis() < stopTime; k++) {
- log.info("Iteration: " + k);
-
- stopAllGrids();
-
- startGrids(3);
-
- ignite = ignite(0);
-
- ignite.getOrCreateCache(ccfg1);
-
- ignite.getOrCreateCache(ccfg2);
-
- ignite.destroyCache(ccfg2.getName());
-
- ignite.getOrCreateCache(ccfg2);
-
- ignite.destroyCache(ccfg1.getName());
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsMultiNodePutGetRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsMultiNodePutGetRestartTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsMultiNodePutGetRestartTest.java
deleted file mode 100644
index d4d9eb6..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsMultiNodePutGetRestartTest.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.File;
-import java.io.Serializable;
-import java.util.List;
-import java.util.UUID;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.cache.query.SqlFieldsQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsMultiNodePutGetRestartTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final int GRID_CNT = 3;
-
- /** */
- private final File allocPath;
-
- /**
- * Default constructor.
- */
- public IgnitePdsMultiNodePutGetRestartTest() {
- String home = U.getIgniteHome();
-
- allocPath = new File(home, "work/db/" + UUID.randomUUID());
-
- allocPath.mkdirs();
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
-
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
- memCfg.setMemoryPolicies(memPlcCfg);
-
- cfg.setMemoryConfiguration(memCfg);
-
- CacheConfiguration ccfg = new CacheConfiguration();
-
- ccfg.setIndexedTypes(Integer.class, DbValue.class);
-
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
-
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
-
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- cfg.setCacheConfiguration(ccfg);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration());
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(IP_FINDER);
-
- cfg.setDiscoverySpi(discoSpi);
-
- cfg.setMarshaller(null);
-
- BinaryConfiguration bCfg = new BinaryConfiguration();
-
- bCfg.setCompactFooter(false);
-
- cfg.setBinaryConfiguration(bCfg);
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPutGetSimple() throws Exception {
- info(">>> Will use path: " + allocPath);
-
- startGrids(GRID_CNT);
-
- try {
- IgniteEx ig = grid(0);
-
- checkPutGetSql(ig, true);
- }
- finally {
- stopAllGrids();
- }
-
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
- info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>");
-
- startGrids(GRID_CNT);
-
- try {
- IgniteEx ig = grid(0);
-
- checkPutGetSql(ig, false);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- * @param ig Ig.
- * @param write Write.
- */
- private void checkPutGetSql(IgniteEx ig, boolean write) {
- IgniteCache<Integer, DbValue> cache = ig.cache(null);
-
- if (write) {
- try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(null)) {
- for (int i = 0; i < 10_000; i++)
- streamer.addData(i, new DbValue(i, "value-" + i, i));
- }
- }
-
- List<List<?>> res = cache.query(new SqlFieldsQuery("select ival from dbvalue where ival < ? order by ival asc")
- .setArgs(10_000)).getAll();
-
- assertEquals(10_000, res.size());
-
- for (int i = 0; i < 10_000; i++) {
- assertEquals(1, res.get(i).size());
- assertEquals(i, res.get(i).get(0));
- }
-
- assertEquals(1, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival = 7899")).getAll().size());
- assertEquals(5000, cache.query(new SqlFieldsQuery("select lval from dbvalue where ival >= 5000 and ival < 10000"))
- .getAll().size());
-
- for (int i = 0; i < 10_000; i++)
- assertEquals(new DbValue(i, "value-" + i, i), cache.get(i));
- }
-
- /**
- *
- */
- private static class DbValue implements Serializable {
- /** */
- @QuerySqlField(index = true)
- private int iVal;
-
- /** */
- @QuerySqlField(index = true)
- private String sVal;
-
- /** */
- @QuerySqlField
- private long lVal;
-
- /**
- * @param iVal Integer value.
- * @param sVal String value.
- * @param lVal Long value.
- */
- public DbValue(int iVal, String sVal, long lVal) {
- this.iVal = iVal;
- this.sVal = sVal;
- this.lVal = lVal;
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- DbValue dbVal = (DbValue)o;
-
- return iVal == dbVal.iVal && lVal == dbVal.lVal &&
- !(sVal != null ? !sVal.equals(dbVal.sVal) : dbVal.sVal != null);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- int res = iVal;
-
- res = 31 * res + (sVal != null ? sVal.hashCode() : 0);
- res = 31 * res + (int)(lVal ^ (lVal >>> 32));
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(DbValue.class, this);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsPageSizesTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsPageSizesTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsPageSizesTest.java
deleted file mode 100644
index a15b96c..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsPageSizesTest.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsPageSizesTest extends GridCommonAbstractTest {
- /** Cache name. */
- private final String cacheName = "cache";
-
- /** */
- private int pageSize;
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
-
- memCfg.setMemoryPolicies(memPlcCfg);
- memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- memCfg.setPageSize(pageSize);
-
- cfg.setMemoryConfiguration(memCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setCacheConfiguration(
- new CacheConfiguration(cacheName)
- .setAffinity(new RendezvousAffinityFunction(false, 32))
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageSize_1k() throws Exception {
- checkPageSize(1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageSize_2k() throws Exception {
- checkPageSize(2 * 1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageSize_4k() throws Exception {
- checkPageSize(4 * 1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageSize_8k() throws Exception {
- checkPageSize(8 * 1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageSize_16k() throws Exception {
- checkPageSize(16 * 1024);
- }
-
- /**
- * @throws Exception if failed.
- */
- private void checkPageSize(int pageSize) throws Exception {
- this.pageSize = pageSize;
-
- IgniteEx ignite = startGrid(0);
-
- try {
- final IgniteCache<Object, Object> cache = ignite.cache(cacheName);
- final long endTime = System.currentTimeMillis() + 60_000;
-
- GridTestUtils.runMultiThreaded(new Callable<Object>() {
- @Override public Object call() throws Exception {
- Random rnd = ThreadLocalRandom.current();
-
- while (System.currentTimeMillis() < endTime) {
- for (int i = 0; i < 500; i++)
- cache.put(rnd.nextInt(100_000), rnd.nextInt());
- }
-
- return null;
- }
- }, 16, "runner");
- }
- finally {
- stopAllGrids();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRecoveryAfterFileCorruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRecoveryAfterFileCorruptionTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRecoveryAfterFileCorruptionTest.java
deleted file mode 100644
index eb3662d..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRecoveryAfterFileCorruptionTest.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.FileChannel;
-import java.util.Collection;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.pagemem.store.PageStore;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.processors.cache.database.file.FilePageStore;
-import org.apache.ignite.internal.processors.cache.database.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsRecoveryAfterFileCorruptionTest extends GridCommonAbstractTest {
- /** Ip finder. */
- private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
-
- /** Total pages. */
- private static final int totalPages = 1024;
-
- /** Cache name. */
- private final String cacheName = "cache";
-
- /** Policy name. */
- private final String policyName = "dfltMemPlc";
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- CacheConfiguration ccfg = new CacheConfiguration(cacheName);
- ccfg.setAffinity(new RendezvousAffinityFunction(true, 1));
-
- ccfg.setRebalanceMode(CacheRebalanceMode.NONE);
-
- cfg.setCacheConfiguration(ccfg);
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName(policyName);
- memPlcCfg.setInitialSize(1024 * 1024 * 1024);
- memPlcCfg.setMaxSize(1024 * 1024 * 1024);
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName(policyName);
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setCheckpointingFrequency(500)
- .setAlwaysWriteFullPages(true)
- );
-
- cfg.setDiscoverySpi(
- new TcpDiscoverySpi()
- .setIpFinder(ipFinder)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- deleteWorkFiles();
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPageRecoveryAfterFileCorruption() throws Exception {
- IgniteEx ig = startGrid(0);
-
- IgniteCache<Integer, Integer> cache = ig.cache(cacheName);
-
- // Put for create data store and init meta page.
- cache.put(1, 1);
-
- GridCacheSharedContext sharedCtx = ig.context().cache().context();
-
- GridCacheDatabaseSharedManager psMgr = (GridCacheDatabaseSharedManager)sharedCtx.database();
-
- FilePageStoreManager pageStore = (FilePageStoreManager)sharedCtx.pageStore();
-
- U.sleep(1_000);
-
- // Disable integrated checkpoint thread.
- psMgr.enableCheckpoints(false).get();
-
- PageMemory mem = sharedCtx.database().memoryPolicy(policyName).pageMemory();
-
- int cacheId = sharedCtx.cache().cache(cacheName).context().cacheId();
-
- FullPageId[] pages = new FullPageId[totalPages];
-
- for (int i = 0; i < totalPages; i++)
- pages[i] = new FullPageId(mem.allocatePage(cacheId, 0, PageIdAllocator.FLAG_DATA), cacheId);
-
- generateWal(
- (PageMemoryImpl)mem,
- sharedCtx.pageStore(),
- sharedCtx.wal(),
- cacheId,
- pages
- );
-
- eraseDataFromDisk(pageStore, cacheId, pages[0]);
-
- stopAllGrids();
-
- ig = startGrid(0);
-
- checkRestore(ig, pages);
- }
-
- /**
- * @param pageStore Page store.
- * @param cacheId Cache id.
- * @param page Page.
- */
- private void eraseDataFromDisk(
- FilePageStoreManager pageStore,
- int cacheId,
- FullPageId page
- ) throws IgniteCheckedException, IOException {
- PageStore store = pageStore.getStore(
- cacheId,
- PageIdUtils.partId(page.pageId())
- );
-
- FilePageStore filePageStore = (FilePageStore)store;
-
- FileChannel ch = U.field(filePageStore, "ch");
-
- long size = ch.size();
-
- ch.write(ByteBuffer.allocate((int)size - FilePageStore.HEADER_SIZE), FilePageStore.HEADER_SIZE);
-
- ch.force(false);
- }
-
- /**
- * @param ig Ig.
- * @param pages Pages.
- */
- private void checkRestore(IgniteEx ig, FullPageId[] pages) throws IgniteCheckedException {
- GridCacheSharedContext<Object, Object> shared = ig.context().cache().context();
-
- GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)shared.database();
-
- dbMgr.enableCheckpoints(false).get();
-
- PageMemory mem = shared.database().memoryPolicy(null).pageMemory();
-
- for (FullPageId fullId : pages) {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- long pageAddr = mem.readLock(fullId.cacheId(), fullId.pageId(), page);
-
- for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
- assertEquals(j + (int)fullId.pageId(), PageUtils.getInt(pageAddr, j));
-
- mem.readUnlock(fullId.cacheId(), fullId.pageId(), page);
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- }
-
- /**
- * @param mem Mem.
- * @param storeMgr Store manager.
- * @param wal Wal.
- * @param cacheId Cache id.
- * @param pages Pages.
- */
- private void generateWal(
- final PageMemoryImpl mem,
- final IgnitePageStoreManager storeMgr,
- final IgniteWriteAheadLogManager wal,
- final int cacheId, FullPageId[] pages
- ) throws Exception {
- // Mark the start position.
- CheckpointRecord cpRec = new CheckpointRecord(null, false);
-
- WALPointer start = wal.log(cpRec);
-
- wal.fsync(start);
-
- for (int i = 0; i < totalPages; i++) {
- FullPageId fullId = pages[i];
-
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- long pageAddr = mem.writeLock(fullId.cacheId(), fullId.pageId(), page);
-
- PageIO.setPageId(pageAddr, fullId.pageId());
-
- try {
- for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
- PageUtils.putInt(pageAddr, j, j + (int)fullId.pageId());
- }
- finally {
- mem.writeUnlock(fullId.cacheId(), fullId.pageId(), page, null, true);
- }
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
-
- Collection<FullPageId> pageIds = mem.beginCheckpoint();
-
- info("Acquired pages for checkpoint: " + pageIds.size());
-
- try {
- ByteBuffer tmpBuf = ByteBuffer.allocate(mem.pageSize());
-
- tmpBuf.order(ByteOrder.nativeOrder());
-
- long begin = System.currentTimeMillis();
-
- long cp = 0;
-
- long write = 0;
-
- for (int i = 0; i < totalPages; i++) {
- FullPageId fullId = pages[i];
-
- if (pageIds.contains(fullId)) {
- long cpStart = System.nanoTime();
-
- Integer tag = mem.getForCheckpoint(fullId, tmpBuf, null);
-
- if (tag == null)
- continue;
-
- long cpEnd = System.nanoTime();
-
- cp += cpEnd - cpStart;
- tmpBuf.rewind();
-
- for (int j = PageIO.COMMON_HEADER_END; j < mem.pageSize(); j += 4)
- assertEquals(j + (int)fullId.pageId(), tmpBuf.getInt(j));
-
- tmpBuf.rewind();
-
- long writeStart = System.nanoTime();
-
- storeMgr.write(cacheId, fullId.pageId(), tmpBuf, tag);
-
- long writeEnd = System.nanoTime();
-
- write += writeEnd - writeStart;
-
- tmpBuf.rewind();
- }
- }
-
- long syncStart = System.currentTimeMillis();
-
- storeMgr.sync(cacheId, 0);
-
- long end = System.currentTimeMillis();
-
- info("Written pages in " + (end - begin) + "ms, copy took " + (cp / 1_000_000) + "ms, " +
- "write took " + (write / 1_000_000) + "ms, sync took " + (end - syncStart) + "ms");
- }
- finally {
- info("Finishing checkpoint...");
-
- mem.finishCheckpoint();
-
- info("Finished checkpoint");
- }
-
- wal.fsync(wal.log(new CheckpointRecord(null, false)));
-
- for (FullPageId fullId : pages) {
- long page = mem.acquirePage(fullId.cacheId(), fullId.pageId());
-
- try {
- assertFalse("Page has a temp heap copy after the last checkpoint: [cacheId=" +
- fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.hasTempCopy(page));
-
- assertFalse("Page is dirty after the last checkpoint: [cacheId=" +
- fullId.cacheId() + ", pageId=" + fullId.pageId() + "]", mem.isDirty(fullId.cacheId(), fullId.pageId(), page));
- }
- finally {
- mem.releasePage(fullId.cacheId(), fullId.pageId(), page);
- }
- }
- }
-
- /**
- *
- */
- private void deleteWorkFiles() throws IgniteCheckedException {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRemoveDuringRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRemoveDuringRebalancingTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRemoveDuringRebalancingTest.java
deleted file mode 100644
index 79d8e40..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsRemoveDuringRebalancingTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.File;
-import java.util.concurrent.Callable;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.IgniteDataStreamer;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.util.typedef.G;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-/**
- *
- */
-public class IgnitePdsRemoveDuringRebalancingTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setCacheConfiguration(
- new CacheConfiguration()
- .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)
- .setBackups(1)
- .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC)
- .setRebalanceMode(CacheRebalanceMode.SYNC)
- );
-
- MemoryConfiguration dbCfg = new MemoryConfiguration();
-
- dbCfg.setConcurrencyLevel(Runtime.getRuntime().availableProcessors() * 4);
- dbCfg.setPageSize(1024);
-
- MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
-
- memPlcCfg.setName("dfltMemPlc");
- memPlcCfg.setInitialSize(100 * 1024 * 1024);
- memPlcCfg.setMaxSize(100 * 1024 * 1024);
- memPlcCfg.setSwapFilePath("db");
-
- dbCfg.setMemoryPolicies(memPlcCfg);
- dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
-
- cfg.setMemoryConfiguration(dbCfg);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- cfg.setDiscoverySpi(
- new TcpDiscoverySpi()
- .setIpFinder(IP_FINDER)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- stopAllGrids();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- U.delete(new File(U.getIgniteHome(), "db"));
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- G.stopAll(true);
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- U.delete(new File(U.getIgniteHome(), "db"));
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testRemovesDuringRebalancing() throws Exception {
- IgniteEx ig = startGrid(0);
-
- try (IgniteDataStreamer<Object, Object> streamer = ig.dataStreamer(null)) {
- streamer.allowOverwrite(true);
-
- for (int i = 0; i < 100_000; i++)
- streamer.addData(i, i);
- }
-
- final IgniteCache<Object, Object> cache = ig.cache(null);
-
- IgniteInternalFuture<Object> fut = GridTestUtils.runAsync(new Callable<Object>() {
- @Override public Object call() throws Exception {
- return startGrid(1);
- }
- });
-
- for (int i = 0; i < 100_000; i++)
- cache.remove(i);
-
- fut.get();
-
- IgniteEx another = grid(1);
-
- IgniteCache<Object, Object> cache1 = another.cache(null);
-
- for (int i = 0; i < 100_000; i++)
- assertNull(cache1.localPeek(i));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodePutGetPersistenceTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodePutGetPersistenceTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodePutGetPersistenceTest.java
deleted file mode 100644
index d92ac25..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodePutGetPersistenceTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class IgnitePdsSingleNodePutGetPersistenceTest extends IgniteDbSingleNodePutGetTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- super.beforeTest();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java
deleted file mode 100644
index 257e0e8..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.cache.affinity.AffinityFunction;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-
-/**
- *
- */
-public class IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest
- extends IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest {
- /** {@inheritDoc} */
- @Override protected void configure(IgniteConfiguration cfg) {
- super.configure(cfg);
-
- for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) {
- AffinityFunction aff = ccfg.getAffinity();
-
- int parts = aff != null ? aff.partitions() : RendezvousAffinityFunction.DFLT_PARTITION_COUNT;
-
- ccfg.setGroupName("testGroup-parts" + parts);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java
deleted file mode 100644
index a18ba75..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest extends IgniteDbSingleNodeWithIndexingPutGetTest {
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setPersistentStoreConfiguration(
- new PersistentStoreConfiguration()
- .setWalMode(WALMode.LOG_ONLY)
- );
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTestsStarted() throws Exception {
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
-
- super.beforeTestsStarted();
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- super.afterTestsStopped();
-
- deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsTxCacheRebalancingTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsTxCacheRebalancingTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsTxCacheRebalancingTest.java
deleted file mode 100644
index f29807e..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePdsTxCacheRebalancingTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.CacheRebalanceMode;
-import org.apache.ignite.cache.CacheWriteSynchronizationMode;
-import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
-import org.apache.ignite.configuration.CacheConfiguration;
-
-/**
- *
- */
-public class IgnitePdsTxCacheRebalancingTest extends IgnitePdsCacheRebalancingAbstractTest {
- /** {@inheritDoc} */
- @Override protected CacheConfiguration cacheConfiguration(String cacheName) {
- CacheConfiguration ccfg = new CacheConfiguration(cacheName);
-
- ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
- ccfg.setCacheMode(CacheMode.PARTITIONED);
- ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
- ccfg.setBackups(1);
- ccfg.setRebalanceDelay(10_000);
- ccfg.setAffinity(new RendezvousAffinityFunction(false, 32));
- ccfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
-
- return ccfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- super.afterTest();
-
- explicitTx = false;
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTopologyChangesWithConstantLoadExplicitTx() throws Exception {
- explicitTx = true;
-
- testTopologyChangesWithConstantLoad();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistenceMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistenceMetricsSelfTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistenceMetricsSelfTest.java
deleted file mode 100644
index 79d4a4d..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistenceMetricsSelfTest.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.Serializable;
-import java.util.Objects;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.MemoryMetrics;
-import org.apache.ignite.PersistenceMetrics;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.PAX;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-
-/**
- *
- */
-public class IgnitePersistenceMetricsSelfTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final String GROUP1 = "grp1";
-
- /** */
- private boolean activeOnStart = true;
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- GridTestUtils.deleteDbFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- GridTestUtils.deleteDbFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setConsistentId(gridName);
-
- cfg.setActiveOnStart(activeOnStart);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
- memCfg.setPageSize(1024);
-
- memCfg.setDefaultMemoryPolicyName("dflt-plc");
-
- MemoryPolicyConfiguration memPlc = new MemoryPolicyConfiguration();
- memPlc.setName("dflt-plc");
- memPlc.setMaxSize(10 * 1024 * 1024);
- memPlc.setMetricsEnabled(true);
-
- memCfg.setMemoryPolicies(memPlc);
-
- cfg.setMemoryConfiguration(memCfg);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration()
- .setMetricsEnabled(true).setWalMode(WALMode.LOG_ONLY));
-
- cfg.setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(false));
-
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
-
- cfg.setCacheConfiguration(cacheConfiguration(GROUP1, "cache", PARTITIONED, ATOMIC, 1));
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- GridTestUtils.deleteDbFiles();
-
- super.afterTest();
- }
-
- /**
- * @param grpName Cache group name.
- * @param name Cache name.
- * @param cacheMode Cache mode.
- * @param atomicityMode Atomicity mode.
- * @param backups Backups number.
- * @return Cache configuration.
- */
- private CacheConfiguration cacheConfiguration(
- String grpName,
- String name,
- CacheMode cacheMode,
- CacheAtomicityMode atomicityMode,
- int backups
- ) {
- CacheConfiguration ccfg = new CacheConfiguration();
-
- ccfg.setName(name);
- ccfg.setGroupName(grpName);
- ccfg.setAtomicityMode(atomicityMode);
- ccfg.setBackups(backups);
- ccfg.setCacheMode(cacheMode);
- ccfg.setWriteSynchronizationMode(FULL_SYNC);
-
- return ccfg;
- }
-
- /**
- * @throws Exception if failed.
- */
- public void testPersistenceMetrics() throws Exception {
- final IgniteEx ig = startGrid(0);
-
- try {
- IgniteCache<Object, Object> cache = ig.cache("cache");
-
- for (int i = 0; i < 10; i++)
- cache.put(i, new Person("first-" + i, "last-" + i));
-
- {
- MemoryMetrics memMetrics = ig.memoryMetrics("dflt-plc");
-
- assertNotNull(memMetrics);
- assertTrue(memMetrics.getDirtyPages() > 0);
- }
-
- ig.context().cache().context().database().waitForCheckpoint("test");
-
- GridTestUtils.waitForCondition(new PAX() {
- @Override public boolean applyx() {
- PersistenceMetrics pMetrics = ig.persistentStoreMetrics();
-
- assertNotNull(pMetrics);
-
- return pMetrics.getLastCheckpointTotalPagesNumber() != 0 &&
- pMetrics.getLastCheckpointDataPagesNumber() != 0;
- }
- }, 5_000);
- }
- finally {
- stopAllGrids();
- }
- }
-
- /**
- *
- */
- static class Person implements Serializable {
- /** */
- @GridToStringInclude
- @QuerySqlField(index = true, groups = "full_name")
- private String fName;
-
- /** */
- @GridToStringInclude
- @QuerySqlField(index = true, groups = "full_name")
- private String lName;
-
- /**
- * @param fName First name.
- * @param lName Last name.
- */
- public Person(String fName, String lName) {
- this.fName = fName;
- this.lName = lName;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(Person.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- Person person = (Person)o;
-
- return Objects.equals(fName, person.fName) &&
- Objects.equals(lName, person.lName);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return Objects.hash(fName, lName);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreCacheGroupsTest.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreCacheGroupsTest.java b/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreCacheGroupsTest.java
deleted file mode 100644
index 4a0ae15..0000000
--- a/modules/pds/src/test/java/org/apache/ignite/cache/database/IgnitePersistentStoreCacheGroupsTest.java
+++ /dev/null
@@ -1,508 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.cache.database;
-
-import java.io.Serializable;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-import java.util.Random;
-import java.util.concurrent.ThreadLocalRandom;
-import javax.cache.Cache;
-import javax.cache.expiry.ExpiryPolicy;
-import org.apache.ignite.Ignite;
-import org.apache.ignite.IgniteCache;
-import org.apache.ignite.cache.CacheAtomicityMode;
-import org.apache.ignite.cache.CacheMode;
-import org.apache.ignite.cache.query.SqlQuery;
-import org.apache.ignite.cache.query.annotations.QuerySqlField;
-import org.apache.ignite.configuration.BinaryConfiguration;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.configuration.WALMode;
-import org.apache.ignite.internal.processors.platform.cache.expiry.PlatformExpiryPolicy;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
-import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC;
-import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
-import static org.apache.ignite.cache.CacheMode.PARTITIONED;
-import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-
-/**
- *
- */
-public class IgnitePersistentStoreCacheGroupsTest extends GridCommonAbstractTest {
- /** */
- private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
-
- /** */
- private static final String GROUP1 = "grp1";
-
- /** */
- private static final String GROUP2 = "grp2";
-
- /** */
- private CacheConfiguration[] ccfgs;
-
- /** */
- private boolean activeOnStart = true;
-
- /** {@inheritDoc} */
- @Override protected void afterTestsStopped() throws Exception {
- GridTestUtils.deleteDbFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected void beforeTest() throws Exception {
- super.beforeTest();
-
- GridTestUtils.deleteDbFiles();
- }
-
- /** {@inheritDoc} */
- @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
- IgniteConfiguration cfg = super.getConfiguration(gridName);
-
- cfg.setConsistentId(gridName);
-
- cfg.setActiveOnStart(activeOnStart);
-
- MemoryConfiguration memCfg = new MemoryConfiguration();
- memCfg.setPageSize(1024);
- memCfg.setDefaultMemoryPolicySize(10 * 1024 * 1024);
-
- cfg.setMemoryConfiguration(memCfg);
-
- cfg.setPersistentStoreConfiguration(new PersistentStoreConfiguration().setWalMode(WALMode.LOG_ONLY));
-
- cfg.setBinaryConfiguration(new BinaryConfiguration().setCompactFooter(false));
-
- ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
-
- if (ccfgs != null) {
- cfg.setCacheConfiguration(ccfgs);
-
- ccfgs = null;
- }
-
- return cfg;
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- stopAllGrids();
-
- GridTestUtils.deleteDbFiles();
-
- super.afterTest();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testClusterRestartStaticCaches1() throws Exception {
- clusterRestart(1, true);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testClusterRestartStaticCaches2() throws Exception {
- clusterRestart(3, true);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testClusterRestartDynamicCaches1() throws Exception {
- clusterRestart(1, false);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testClusterRestartDynamicCaches2() throws Exception {
- clusterRestart(3, false);
- }
-
- /**
- * @throws Exception If failed.
- */
- @SuppressWarnings("unchecked")
- public void testClusterRestartCachesWithH2Indexes() throws Exception {
- CacheConfiguration[] ccfgs1 = new CacheConfiguration[5];
-
- // Several caches with the same indexed type (and index names).
- ccfgs1[0] = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1).
- setIndexedTypes(Integer.class, Person.class);
- ccfgs1[1] = cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL, 1).
- setIndexedTypes(Integer.class, Person.class);
- ccfgs1[2] = cacheConfiguration(GROUP2, "c3", PARTITIONED, ATOMIC, 1).
- setIndexedTypes(Integer.class, Person.class);
- ccfgs1[3] = cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 1).
- setIndexedTypes(Integer.class, Person.class);
- ccfgs1[4] = cacheConfiguration(null, "c5", PARTITIONED, ATOMIC, 1).
- setIndexedTypes(Integer.class, Person.class);
-
- String[] caches = {"c1", "c2", "c3", "c4", "c5"};
-
- startGrids(3);
-
- Ignite node = ignite(0);
-
- node.createCaches(Arrays.asList(ccfgs1));
-
- putPersons(caches, node);
-
- checkPersons(caches, node);
- checkPersonsQuery(caches, node);
-
- stopAllGrids();
-
- startGrids(3);
-
- awaitPartitionMapExchange();
-
- node = ignite(0);
-
- checkPersons(caches, node);
- checkPersonsQuery(caches, node);
-
- Random rnd = ThreadLocalRandom.current();
-
- int idx = rnd.nextInt(caches.length);
-
- String cacheName = caches[idx];
- CacheConfiguration cacheCfg = ccfgs1[idx];
-
- node.destroyCache(cacheName);
-
- node.createCache(cacheCfg);
-
- putPersons(new String[]{cacheName}, node);
-
- checkPersons(caches, node);
- checkPersonsQuery(caches, node);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void _testExpiryPolicy() throws Exception {
- long ttl = 10000;
-
- activeOnStart = false;
-
- CacheConfiguration[] ccfgs1 = new CacheConfiguration[5];
-
- ccfgs1[0] = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1);
- ccfgs1[1] = cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL, 1);
- ccfgs1[2] = cacheConfiguration(GROUP2, "c3", PARTITIONED, ATOMIC, 1);
- ccfgs1[3] = cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 1);
- ccfgs1[4] = cacheConfiguration(null, "c5", PARTITIONED, ATOMIC, 1);
-
- String[] caches = {"c1", "c2", "c3", "c4", "c5"};
-
- startGrids(3);
-
- Ignite node = ignite(0);
-
- node.active(true);
-
- node.createCaches(Arrays.asList(ccfgs1));
-
- ExpiryPolicy plc = new PlatformExpiryPolicy(ttl, -2, -2);
-
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName).withExpiryPolicy(plc);
-
- for (int i = 0; i < 10; i++)
- cache.put(i, cacheName + i);
- }
-
- long deadline = System.currentTimeMillis() + (long)(ttl * 1.2);
-
- stopAllGrids();
-
- startGrids(3);
-
- node = ignite(0);
-
- node.active(true);
-
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- for (int i = 0; i < 10; i++)
- assertEquals(cacheName + i, cache.get(i));
-
- assertEquals(10, cache.size());
- }
-
- // Wait for expiration.
- Thread.sleep(Math.max(deadline - System.currentTimeMillis(), 0));
-
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- assertEquals(0, cache.size());
- }
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testCreateDropCache() throws Exception {
- ccfgs = new CacheConfiguration[]{cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1)
- .setIndexedTypes(Integer.class, Person.class)};
-
- Ignite ignite = startGrid();
-
- ignite.cache("c1").destroy();
-
- stopGrid();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testCreateDropCache1() throws Exception {
- CacheConfiguration ccfg1 = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1);
-
- CacheConfiguration ccfg2 = cacheConfiguration(GROUP1, "c2", PARTITIONED, ATOMIC, 1);
-
- Ignite ignite = startGrid();
-
- ignite.createCaches(Arrays.asList(ccfg1, ccfg2));
-
-
- ignite.cache("c1").destroy();
-
- ignite.cache("c2").destroy();
-
- ignite.createCache(ccfg1);
- ignite.createCache(ccfg2);
-
- stopGrid();
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testCreateDropCache2() throws Exception {
- CacheConfiguration ccfg1 = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1)
- .setIndexedTypes(Integer.class, Person.class);
-
- CacheConfiguration ccfg2 = cacheConfiguration(GROUP1, "c2", PARTITIONED, ATOMIC, 1)
- .setIndexedTypes(Integer.class, Person.class);
-
- Ignite ignite = startGrid();
-
- ignite.createCaches(Arrays.asList(ccfg1, ccfg2));
-
- ignite.cache("c1").destroy();
-
- ignite.createCache(ccfg1);
-
- stopGrid();
- }
-
- /**
- * @param caches Cache names to put data into.
- * @param node Ignite node.
- */
- private void putPersons(String[] caches, Ignite node) {
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- for (int i = 0; i < 10; i++)
- cache.put(i, new Person("" + i, cacheName));
- }
- }
-
- /**
- * @param caches Cache names to invoke a query against to.
- * @param node Ignite node.
- */
- private void checkPersons(String[] caches, Ignite node) {
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- for (int i = 0; i < 10; i++)
- assertEquals(new Person("" + i, cacheName), cache.get(i));
-
- assertEquals(10, cache.size());
- }
- }
-
- /**
- * @param caches Cache names to invoke a query against to.
- * @param node Ignite node.
- */
- private void checkPersonsQuery(String[] caches, Ignite node) {
- SqlQuery<Integer, Person> qry = new SqlQuery<>(
- Person.class, "SELECT p.* FROM Person p WHERE p.lname=? ORDER BY p.fname");
-
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- List<Cache.Entry<Integer, Person>> persons = cache.query(qry.setArgs(cacheName)).getAll();
-
- for (int i = 0; i < 10; i++)
- assertEquals(new Person("" + i, cacheName), persons.get(i).getValue());
-
- assertEquals(10, persons.size());
- }
- }
-
- /**
- * @param nodes Nodes number.
- * @param staticCaches {@code True} if caches should be statically configured.
- * @throws Exception If failed.
- */
- private void clusterRestart(int nodes, boolean staticCaches) throws Exception {
- CacheConfiguration[] ccfgs = new CacheConfiguration[5];
-
- ccfgs[0] = cacheConfiguration(GROUP1, "c1", PARTITIONED, ATOMIC, 1);
- ccfgs[1] = cacheConfiguration(GROUP1, "c2", PARTITIONED, TRANSACTIONAL, 1);
- ccfgs[2] = cacheConfiguration(GROUP2, "c3", PARTITIONED, ATOMIC, 1);
- ccfgs[3] = cacheConfiguration(GROUP2, "c4", PARTITIONED, TRANSACTIONAL, 1);
- ccfgs[4] = cacheConfiguration(null, "c5", PARTITIONED, ATOMIC, 1);
-
- String[] caches = {"c1", "c2", "c3", "c4", "c5"};
-
- for (int i = 0; i < nodes; i++) {
- if (staticCaches)
- this.ccfgs = ccfgs;
-
- startGrid(i);
- }
-
- Ignite node = ignite(0);
-
- if (!staticCaches)
- node.createCaches(Arrays.asList(ccfgs));
-
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- for (int i = 0; i < 10; i++) {
- cache.put(i, cacheName + i);
-
- assertEquals(cacheName + i, cache.get(i));
- }
-
- assertEquals(10, cache.size());
- }
-
- stopAllGrids();
-
- node = startGrids(nodes);
-
- awaitPartitionMapExchange();
-
- for (String cacheName : caches) {
- IgniteCache<Object, Object> cache = node.cache(cacheName);
-
- for (int i = 0; i < 10; i++)
- assertEquals(cacheName + i, cache.get(i));
-
- assertEquals(10, cache.size());
- }
- }
-
- /**
- * @param grpName Cache group name.
- * @param name Cache name.
- * @param cacheMode Cache mode.
- * @param atomicityMode Atomicity mode.
- * @param backups Backups number.
- * @return Cache configuration.
- */
- private CacheConfiguration cacheConfiguration(
- String grpName,
- String name,
- CacheMode cacheMode,
- CacheAtomicityMode atomicityMode,
- int backups
- ) {
- CacheConfiguration ccfg = new CacheConfiguration();
-
- ccfg.setName(name);
- ccfg.setGroupName(grpName);
- ccfg.setAtomicityMode(atomicityMode);
- ccfg.setBackups(backups);
- ccfg.setCacheMode(cacheMode);
- ccfg.setWriteSynchronizationMode(FULL_SYNC);
-
- return ccfg;
- }
-
- /**
- *
- */
- static class Person implements Serializable {
- /** */
- @GridToStringInclude
- @QuerySqlField(index = true, groups = "full_name")
- String fName;
-
- /** */
- @GridToStringInclude
- @QuerySqlField(index = true, groups = "full_name")
- String lName;
-
- /**
- * @param fName First name.
- * @param lName Last name.
- */
- public Person(String fName, String lName) {
- this.fName = fName;
- this.lName = lName;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(Person.class, this);
- }
-
- /** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
- if (o == null || getClass() != o.getClass())
- return false;
- Person person = (Person)o;
- return Objects.equals(fName, person.fName) &&
- Objects.equals(lName, person.lName);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return Objects.hash(fName, lName);
- }
- }
-}
[29/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
new file mode 100644
index 0000000..be1e477
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
@@ -0,0 +1,453 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * File input.
+ */
+public final class FileInput implements ByteBufferBackedDataInput {
+ /** */
+ private ByteBuffer buf;
+
+ /** */
+ private FileChannel ch;
+
+ /** */
+ private long pos;
+
+ /**
+ * @param ch Channel.
+ * @param buf Buffer.
+ */
+ public FileInput(FileChannel ch, ByteBuffer buf) throws IOException {
+ assert ch != null;
+
+ this.ch = ch;
+ this.buf = buf;
+
+ pos = ch.position();
+
+ clearBuffer();
+ }
+
+ /**
+ * Clear buffer.
+ */
+ private void clearBuffer() {
+ buf.clear();
+ buf.limit(0);
+
+ assert buf.remaining() == 0; // Buffer is empty.
+ }
+
+ /**
+ * @param pos Position in bytes from file begin.
+ */
+ public void seek(long pos) throws IOException {
+ if (pos > ch.size())
+ throw new EOFException();
+
+ ch.position(pos);
+
+ this.pos = pos;
+
+ clearBuffer();
+ }
+
+ /**
+ * @return Underlying buffer.
+ */
+ @Override public ByteBuffer buffer() {
+ return buf;
+ }
+
+
+ /** {@inheritDoc} */
+ @Override public void ensure(int requested) throws IOException {
+ int available = buf.remaining();
+
+ if (available >= requested)
+ return;
+
+ if (buf.capacity() < requested)
+ throw new IOException("Requested size is greater than buffer: " + requested);
+
+ buf.compact();
+
+ do {
+ int read = ch.read(buf);
+
+ if (read == -1)
+ throw new EOFException();
+
+ available += read;
+
+ pos += read;
+ }
+ while (available < requested);
+
+ buf.flip();
+ }
+
+ /**
+ * @return Position in the stream.
+ */
+ public long position() {
+ return pos - buf.remaining();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public void readFully(@NotNull byte[] b) throws IOException {
+ ensure(b.length);
+
+ buf.get(b);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+ ensure(len);
+
+ buf.get(b, off, len);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int skipBytes(int n) throws IOException {
+ if (buf.remaining() >= n)
+ buf.position(buf.position() + n);
+ else
+ seek(pos + n);
+
+ return n;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public boolean readBoolean() throws IOException {
+ return readByte() == 1;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public byte readByte() throws IOException {
+ ensure(1);
+
+ return buf.get();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readUnsignedByte() throws IOException {
+ return readByte() & 0xFF;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public short readShort() throws IOException {
+ ensure(2);
+
+ return buf.getShort();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readUnsignedShort() throws IOException {
+ return readShort() & 0xFFFF;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public char readChar() throws IOException {
+ ensure(2);
+
+ return buf.getChar();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readInt() throws IOException {
+ ensure(4);
+
+ return buf.getInt();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public long readLong() throws IOException {
+ ensure(8);
+
+ return buf.getLong();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public float readFloat() throws IOException {
+ ensure(4);
+
+ return buf.getFloat();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public double readDouble() throws IOException {
+ ensure(8);
+
+ return buf.getDouble();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String readLine() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String readUTF() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * @param skipCheck If CRC check should be skipped.
+ * @return autoclosable fileInput, after its closing crc32 will be calculated and compared with saved one
+ */
+ public Crc32CheckingFileInput startRead(boolean skipCheck) {
+ return new Crc32CheckingFileInput(buf.position(), skipCheck);
+ }
+
+ /**
+ * Checking of CRC32.
+ */
+ public class Crc32CheckingFileInput implements ByteBufferBackedDataInput, AutoCloseable {
+ /** */
+ private final PureJavaCrc32 crc32 = new PureJavaCrc32();
+
+ /** Last calc position. */
+ private int lastCalcPosition;
+
+ /** Skip crc check. */
+ private boolean skipCheck;
+
+ /**
+ * @param position Position.
+ */
+ public Crc32CheckingFileInput(int position, boolean skipCheck) {
+ this.lastCalcPosition = position;
+ this.skipCheck = skipCheck;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void ensure(int requested) throws IOException {
+ int available = buf.remaining();
+
+ if (available >= requested)
+ return;
+
+ updateCrc();
+
+ FileInput.this.ensure(requested);
+
+ lastCalcPosition = 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws Exception {
+ updateCrc();
+
+ int val = crc32.getValue();
+
+ int writtenCrc = this.readInt();
+
+ if ((val ^ writtenCrc) != 0 && !skipCheck) {
+ // If it last message we will skip it (EOF will be thrown).
+ ensure(5);
+
+ throw new IgniteDataIntegrityViolationException(
+ "val: " + val + " writtenCrc: " + writtenCrc
+ );
+ }
+ }
+
+ /**
+ *
+ */
+ private void updateCrc() {
+ if (skipCheck)
+ return;
+
+ int oldPos = buf.position();
+
+ buf.position(lastCalcPosition);
+
+ crc32.update(buf, oldPos - lastCalcPosition);
+
+ lastCalcPosition = oldPos;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int skipBytes(int n) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public void readFully(@NotNull byte[] b) throws IOException {
+ ensure(b.length);
+
+ buf.get(b);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public void readFully(@NotNull byte[] b, int off, int len) throws IOException {
+ ensure(len);
+
+ buf.get(b, off, len);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public boolean readBoolean() throws IOException {
+ return readByte() == 1;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public byte readByte() throws IOException {
+ ensure(1);
+
+ return buf.get();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readUnsignedByte() throws IOException {
+ return readByte() & 0xFF;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public short readShort() throws IOException {
+ ensure(2);
+
+ return buf.getShort();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readUnsignedShort() throws IOException {
+ return readShort() & 0xFFFF;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public char readChar() throws IOException {
+ ensure(2);
+
+ return buf.getChar();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public int readInt() throws IOException {
+ ensure(4);
+
+ return buf.getInt();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public long readLong() throws IOException {
+ ensure(8);
+
+ return buf.getLong();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public float readFloat() throws IOException {
+ ensure(4);
+
+ return buf.getFloat();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public double readDouble() throws IOException {
+ ensure(8);
+
+ return buf.getDouble();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String readLine() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override public String readUTF() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ /** {@inheritDoc} */
+ @Override public ByteBuffer buffer() {
+ return FileInput.this.buffer();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java
new file mode 100644
index 0000000..033b1c4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWALPointer.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.util.typedef.internal.S;
+
+/**
+ * File WAL pointer.
+ */
+public class FileWALPointer implements WALPointer, Comparable<FileWALPointer> {
+ /** */
+ private final long idx;
+
+ /** */
+ private final int fileOffset;
+
+ /** Written record length */
+ private int len;
+
+ /** Force flush flag. Used in BACKGROUND WAL mode. */
+ private boolean forceFlush;
+
+ /**
+ * @param idx File timestamp index.
+ * @param fileOffset Offset in file, from the beginning.
+ * @param len Record length.
+ */
+ public FileWALPointer(long idx, int fileOffset, int len) {
+ this(idx, fileOffset, len, false);
+ }
+
+ /**
+ * @param idx File timestamp index.
+ * @param fileOffset Offset in file, from the beginning.
+ * @param len Record length.
+ * @param forceFlush Force flush flag.
+ */
+ public FileWALPointer(long idx, int fileOffset, int len, boolean forceFlush) {
+ this.idx = idx;
+ this.fileOffset = fileOffset;
+ this.len = len;
+ this.forceFlush = forceFlush;
+ }
+
+ /**
+ * @return Timestamp index.
+ */
+ public long index() {
+ return idx;
+ }
+
+ /**
+ * @return File offset.
+ */
+ public int fileOffset() {
+ return fileOffset;
+ }
+
+ /**
+ * @return Record length.
+ */
+ public int length() {
+ return len;
+ }
+
+ /**
+ * @param len Record length.
+ */
+ public void length(int len) {
+ this.len = len;
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALPointer next() {
+ if (len == 0)
+ throw new IllegalStateException("Failed to calculate next WAL pointer " +
+ "(this pointer is a terminal): " + this);
+
+ // Return a terminal pointer.
+ return new FileWALPointer(idx, fileOffset + len, 0);
+ }
+
+ /**
+ * @return Force flush flag.
+ */
+ public boolean forceFlush() {
+ return forceFlush;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (!(o instanceof FileWALPointer))
+ return false;
+
+ FileWALPointer that = (FileWALPointer)o;
+
+ return idx == that.idx && fileOffset == that.fileOffset;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = (int)(idx ^ (idx >>> 32));
+
+ result = 31 * result + fileOffset;
+
+ return result;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int compareTo(FileWALPointer o) {
+ int res = Long.compare(idx, o.idx);
+
+ return res == 0 ? Integer.compare(fileOffset, o.fileOffset) : res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(FileWALPointer.class, this);
+ }
+}
[17/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
index 723c596..2245a4d 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbAbstractTest.java
@@ -24,7 +24,7 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
index 0a6e9f8..93e5181 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbMemoryLeakAbstractTest.java
@@ -25,7 +25,7 @@ import org.apache.ignite.configuration.MemoryConfiguration;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
-import org.apache.ignite.internal.processors.cache.database.DataStructure;
+import org.apache.ignite.internal.processors.cache.persistence.DataStructure;
import static org.apache.ignite.IgniteSystemProperties.getInteger;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MemoryMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MemoryMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MemoryMetricsSelfTest.java
index 3a99cb2..f9a9824 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MemoryMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MemoryMetricsSelfTest.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.processors.database;
import java.util.concurrent.CountDownLatch;
import org.apache.ignite.MemoryMetrics;
import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static java.lang.Thread.sleep;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
index 5206472..dcd4ce1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/database/MetadataStorageSelfTest.java
@@ -28,11 +28,11 @@ import org.apache.ignite.internal.mem.DirectMemoryProvider;
import org.apache.ignite.internal.pagemem.FullPageId;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.MetadataStorage;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MetadataStorage;
import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
index 310082d..12200ae 100644
--- a/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
+++ b/modules/core/src/test/java/org/apache/ignite/loadtests/hashmap/GridCacheTestContext.java
@@ -18,7 +18,6 @@
package org.apache.ignite.loadtests.hashmap;
import java.util.IdentityHashMap;
-import java.util.UUID;
import org.apache.ignite.cache.store.CacheStore;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
@@ -35,9 +34,9 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvccManager;
import org.apache.ignite.internal.processors.cache.GridCachePartitionExchangeManager;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridCacheTtlManager;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.GridCacheSharedTtlCleanupManager;
-import org.apache.ignite.internal.processors.cache.database.IgniteCacheSnapshotManager;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheSnapshotManager;
import org.apache.ignite.internal.processors.cache.datastructures.CacheDataStructuresManager;
import org.apache.ignite.internal.processors.cache.dr.GridOsCacheDrManager;
import org.apache.ignite.internal.processors.cache.jta.CacheNoopJtaManager;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
index 943c5f5..46d5498 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite2.java
@@ -45,7 +45,7 @@ import org.apache.ignite.internal.processors.cache.IgniteCachePartitionMapUpdate
import org.apache.ignite.internal.processors.cache.IgniteDynamicCacheAndNodeStop;
import org.apache.ignite.internal.processors.cache.IgniteOnePhaseCommitInvokeTest;
import org.apache.ignite.internal.processors.cache.MemoryPolicyConfigValidationTest;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicyInitializationTest;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryPolicyInitializationTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheLoadingConcurrentGridStartSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.CacheLoadingConcurrentGridStartSelfTestAllowOverwrite;
import org.apache.ignite.internal.processors.cache.distributed.CacheLockReleaseNodeLeaveTest;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java
new file mode 100644
index 0000000..3e040f4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsOutOfMemoryTestSuite.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgnitePdsWalTlbTest;
+
+/**
+ *
+ */
+public class IgnitePdsOutOfMemoryTestSuite extends TestSuite {
+ /**
+ * @return Suite.
+ * @throws Exception If failed.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Persistent Store OOM Test Suite");
+
+ suite.addTestSuite(IgnitePdsWalTlbTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
new file mode 100644
index 0000000..59e7ae9
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsClientNearCachePutGetTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsDynamicCacheTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsSingleNodePutGetPersistenceTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreCacheGroupsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMultiNodePutGetRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheIntegrationTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsEvictionTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.ignitePdsCheckpointSimulationTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalDirectoriesConfigurationTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreePageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTreeReuseListPageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.MetadataStoragePageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
+import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
+import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
+import org.apache.ignite.internal.processors.database.IgniteDbSingleNodePutGetTest;
+import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeTinyPutGetTest;
+
+
+/**
+ *
+ */
+public class IgnitePdsTestSuite extends TestSuite {
+ /**
+ * @return Suite.
+ * @throws Exception If failed.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Persistent Store Test Suite");
+
+ // Basic PageMemory tests.
+ suite.addTestSuite(PageMemoryImplNoLoadTest.class);
+ suite.addTestSuite(MetadataStoragePageMemoryImplTest.class);
+ suite.addTestSuite(IgnitePdsEvictionTest.class);
+ suite.addTestSuite(PageMemoryImplTest.class);
+
+ // Checkpointing smoke-test.
+ suite.addTestSuite(IgnitePdsCacheIntegrationTest.class);
+ suite.addTestSuite(ignitePdsCheckpointSimulationTest.class);
+
+ // BTree tests with store page memory.
+ suite.addTestSuite(BPlusTreePageMemoryImplTest.class);
+ suite.addTestSuite(BPlusTreeReuseListPageMemoryImplTest.class);
+
+ // Basic API tests.
+ suite.addTestSuite(IgniteDbSingleNodePutGetTest.class);
+ suite.addTestSuite(IgniteDbMultiNodePutGetTest.class);
+ suite.addTestSuite(IgniteDbSingleNodeTinyPutGetTest.class);
+ suite.addTestSuite(IgniteDbDynamicCacheSelfTest.class);
+ suite.addTestSuite(IgniteDbClientNearCachePutGetTest.class);
+
+ // Persistence-enabled.
+ suite.addTestSuite(IgnitePdsMultiNodePutGetRestartTest.class);
+ suite.addTestSuite(IgnitePdsSingleNodePutGetPersistenceTest.class);
+ suite.addTestSuite(IgnitePdsPageEvictionTest.class);
+ suite.addTestSuite(IgnitePdsDynamicCacheTest.class);
+ suite.addTestSuite(IgniteWalDirectoriesConfigurationTest.class);
+ suite.addTestSuite(IgnitePdsClientNearCachePutGetTest.class);
+
+ suite.addTestSuite(IgnitePersistentStoreCacheGroupsTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
new file mode 100644
index 0000000..f645dba
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsAtomicCacheRebalancingTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsTxCacheRebalancingTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistenceMetricsSelfTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistentStoreDataStructuresTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionDuringPartitionClearTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsTransactionsHangTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalHistoryReservationsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.crc.IgniteDataIntegrityTests;
+
+/**
+ *
+ */
+public class IgnitePdsTestSuite2 extends TestSuite {
+ /**
+ * @return Suite.
+ * @throws Exception If failed.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite persistent Store Test Suite 2");
+
+ // Integrity test.
+ suite.addTestSuite(IgniteDataIntegrityTests.class);
+ suite.addTestSuite(IgnitePdsRecoveryAfterFileCorruptionTest.class);
+ suite.addTestSuite(IgnitePdsPageSizesTest.class);
+
+ // Metrics test.
+ suite.addTestSuite(IgnitePersistenceMetricsSelfTest.class);
+
+ // WAL recovery test.
+ suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class);
+ suite.addTestSuite(IgniteWalRecoveryTest.class);
+
+ suite.addTestSuite(IgnitePdsTransactionsHangTest.class);
+
+ suite.addTestSuite(IgnitePdsNoActualWalHistoryTest.class);
+
+ suite.addTestSuite(IgnitePdsRebalancingOnNotStableTopologyTest.class);
+
+ suite.addTestSuite(IgnitePdsWholeClusterRestartTest.class);
+
+ suite.addTestSuite(IgnitePdsPageEvictionDuringPartitionClearTest.class);
+
+ // Rebalancing test
+ suite.addTestSuite(IgnitePdsAtomicCacheRebalancingTest.class);
+ suite.addTestSuite(IgnitePdsTxCacheRebalancingTest.class);
+ suite.addTestSuite(IgniteWalHistoryReservationsTest.class);
+
+ suite.addTestSuite(IgnitePdsContinuousRestartTest.class);
+
+ suite.addTestSuite(IgnitePersistentStoreDataStructuresTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
new file mode 100644
index 0000000..b120c62
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteStandByClusterSuite.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateCacheTest;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStreamerTest;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateDataStructureTest;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateFailOverTest;
+import org.apache.ignite.internal.processors.cache.persistence.standbycluster.IgniteChangeGlobalStateTest;
+
+/**
+ *
+ */
+public class IgniteStandByClusterSuite extends TestSuite {
+ /**
+ *
+ */
+ public static TestSuite suite() {
+ TestSuite suite = new TestSuite("Ignite Activate/DeActivate Cluster Test Suit");
+
+ suite.addTestSuite(IgniteChangeGlobalStateTest.class);
+ suite.addTestSuite(IgniteChangeGlobalStateCacheTest.class);
+ suite.addTestSuite(IgniteChangeGlobalStateDataStructureTest.class);
+ suite.addTestSuite(IgniteChangeGlobalStateDataStreamerTest.class);
+ suite.addTestSuite(IgniteChangeGlobalStateFailOverTest.class);
+// suite.addTestSuite(IgniteChangeGlobalStateServiceTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index ddd0c59..2a7e888 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -72,7 +72,7 @@ import org.apache.ignite.internal.processors.cache.KeyCacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.GridCacheEntryRemovedException;
import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvalidPartitionException;
import org.apache.ignite.internal.processors.cache.query.CacheQueryPartitionInfo;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryMarshallable;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
index 0440615..2ae5868 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2PkHashIndex.java
@@ -26,7 +26,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Table;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowFactory.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowFactory.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowFactory.java
index 86b2749..2e57ca3 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowFactory.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2RowFactory.java
@@ -21,7 +21,7 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.internal.pagemem.PageIdUtils;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRowAdapter;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2RowDescriptor;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
index f2b8bad..b60328d 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java
@@ -23,10 +23,10 @@ import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusMetaIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList;
import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO;
import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
index e7ce50e..70d8632 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java
@@ -24,9 +24,9 @@ import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteSystemProperties;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.database.RootPage;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.RootPage;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.query.h2.H2Cursor;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2IndexBase;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java
index 5e6a36f..0e8407c 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java
@@ -20,11 +20,11 @@ package org.apache.ignite.internal.processors.query.h2.database.io;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelper;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java
index c4bb387..4a68739 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java
@@ -20,11 +20,11 @@ package org.apache.ignite.internal.processors.query.h2.database.io;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
import org.apache.ignite.internal.processors.query.h2.database.InlineIndexHelper;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
index 8252a69..4d7b3a2 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2InnerIO.java
@@ -19,10 +19,10 @@ package org.apache.ignite.internal.processors.query.h2.database.io;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusInnerIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
import org.h2.result.SearchRow;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
index a24eb99..f292fc1 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2LeafIO.java
@@ -19,10 +19,10 @@ package org.apache.ignite.internal.processors.query.h2.database.io;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
+import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusLeafIO;
+import org.apache.ignite.internal.processors.cache.persistence.tree.io.IOVersions;
import org.apache.ignite.internal.processors.query.h2.database.H2Tree;
import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row;
import org.h2.result.SearchRow;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
index 7de6740..bd64bce 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Row.java
@@ -19,7 +19,7 @@ package org.apache.ignite.internal.processors.query.h2.opt;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.h2.result.Row;
import org.h2.result.SearchRow;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
index 48a61cc..33a9b85 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgniteCacheLockPartitionOnAffinityRunTest.java
@@ -35,7 +35,7 @@ import org.apache.ignite.compute.ComputeTaskSession;
import org.apache.ignite.internal.IgniteEx;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.util.lang.GridCursor;
import org.apache.ignite.lang.IgniteBiPredicate;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java
new file mode 100644
index 0000000..2d566a0
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.cache.affinity.AffinityFunction;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest
+ extends IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest {
+ /** {@inheritDoc} */
+ @Override protected void configure(IgniteConfiguration cfg) {
+ super.configure(cfg);
+
+ for (CacheConfiguration ccfg : cfg.getCacheConfiguration()) {
+ AffinityFunction aff = ccfg.getAffinity();
+
+ int parts = aff != null ? aff.partitions() : RendezvousAffinityFunction.DFLT_PARTITION_COUNT;
+
+ ccfg.setGroupName("testGroup-parts" + parts);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java
new file mode 100644
index 0000000..7e7b9e6
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
+import org.apache.ignite.internal.util.typedef.internal.U;
+
+/**
+ *
+ */
+public class IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest extends IgniteDbSingleNodeWithIndexingPutGetTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.LOG_ONLY)
+ );
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+
+ super.beforeTestsStarted();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingPutGetTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingPutGetTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingPutGetTest.java
index 46a24a0..2971fff 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingPutGetTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgniteDbSingleNodeWithIndexingPutGetTest.java
@@ -32,7 +32,6 @@ import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
new file mode 100644
index 0000000..927de07
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
@@ -0,0 +1,297 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.database;
+
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.LinkedHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.QueryEntity;
+import org.apache.ignite.cache.QueryIndex;
+import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.query.h2.IgniteH2Indexing;
+import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
+
+/**
+ *
+ */
+public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Index name. */
+ private static final String IDX_NAME = "my_idx";
+
+ /** Cache name. */
+ private static final String TMPL_NAME = "test_cache*";
+
+ /** Table name. */
+ private static final String TBL_NAME = Person.class.getSimpleName();
+
+ /** Schema name. */
+ private static final String SCHEMA_NAME = "PUBLIC";
+
+ /** Cache name. */
+ private static final String CACHE_NAME = TBL_NAME;
+
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ cfg.setCacheConfiguration(cacheCfg(TMPL_NAME));
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ pCfg.setCheckpointingFrequency(1000);
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ return cfg;
+ }
+
+ /** */
+ private CacheConfiguration cacheCfg(String name) {
+ CacheConfiguration<?, ?> cfg = new CacheConfiguration<>();
+
+ cfg.setName(name);
+
+ cfg.setRebalanceMode(CacheRebalanceMode.NONE);
+
+ cfg.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+
+ return cfg;
+ }
+
+ /** */
+ private QueryEntity getEntity() {
+ LinkedHashMap<String, String> fields = new LinkedHashMap<>();
+
+ fields.put("id", Integer.class.getName());
+ fields.put("name", String.class.getName());
+
+ QueryEntity entity = new QueryEntity(Integer.class.getName(), Person.class.getName());
+ entity.setFields(fields);
+ entity.setTableName(TBL_NAME);
+
+ return entity;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ System.setProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK, "true");
+
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ deleteWorkFiles();
+
+ System.clearProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK);
+ }
+
+ /** */
+ public void testPersistIndex() throws Exception {
+ IgniteEx ig0 = startGrid(0);
+ startGrid(1);
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
+
+ db.addCheckpointListener(new DbCheckpointListener() {
+ @Override public void onCheckpointBegin(Context context) {
+ cnt.incrementAndGet();
+ }
+ });
+
+ QueryIndex idx = new QueryIndex("name");
+
+ idx.setName(IDX_NAME);
+
+ ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, 1, true);
+
+ assert indexCnt(ig0, CACHE_NAME) == 0;
+
+ ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get();
+
+ assert indexCnt(ig0, CACHE_NAME) == 1;
+
+ waitForCheckpoint(cnt);
+
+ stopGrid(1);
+
+ IgniteEx ig1 = startGrid(1);
+
+ assert indexCnt(ig1, CACHE_NAME) == 1;
+ }
+
+ /** */
+ public void testPersistCompositeIndex() throws Exception {
+ IgniteEx ig0 = startGrid(0);
+ startGrid(1);
+
+ final AtomicInteger cnt = new AtomicInteger();
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
+
+ db.addCheckpointListener(new DbCheckpointListener() {
+ @Override public void onCheckpointBegin(Context context) {
+ cnt.incrementAndGet();
+ }
+ });
+
+ ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, 1, true);
+
+ assert indexCnt(ig0, CACHE_NAME) == 0;
+
+ QueryIndex idx = new QueryIndex(Arrays.asList("id", "name"), QueryIndexType.SORTED);
+
+ idx.setName(IDX_NAME);
+
+ ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get();
+
+ assert indexCnt(ig0, CACHE_NAME) == 1;
+
+ waitForCheckpoint(cnt);
+
+ stopGrid(1);
+
+ IgniteEx ig1 = startGrid(1);
+
+ assert indexCnt(ig1, CACHE_NAME) == 1;
+ }
+
+ /** */
+ private void waitForCheckpoint(final AtomicInteger cnt) throws IgniteInterruptedCheckedException {
+ final int i = cnt.get();
+
+ GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return cnt.get() > i;
+ }
+ }, 2000);
+ }
+
+ /** */
+ private int indexCnt(IgniteEx node, String cacheName) {
+
+ DynamicCacheDescriptor desc = node.context().cache().cacheDescriptor(cacheName);
+
+ int cnt = 0;
+
+ for (QueryEntity entity : desc.schema().entities()) {
+ cnt += entity.getIndexes().size();
+ }
+
+ return cnt;
+ }
+
+ /**
+ * @return Indexing.
+ */
+ private IgniteH2Indexing getIndexing(IgniteEx ignite) {
+ return U.field(ignite.context().query(), "idx");
+ }
+
+ /**
+ *
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ *
+ */
+ protected static class Person implements Serializable {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ @SuppressWarnings("unused")
+ private Person() {
+ // No-op.
+ }
+
+ /** */
+ public Person(int id) {
+ this.id = id;
+ }
+
+ /** */
+ @QuerySqlField
+ protected int id;
+
+ /** */
+ @QuerySqlField
+ protected String name;
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ IgnitePersistentStoreSchemaLoadTest.Person person = (IgnitePersistentStoreSchemaLoadTest.Person)o;
+
+ if (id != person.id)
+ return false;
+
+ return name != null ? name.equals(person.name) : person.name == null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int result = id;
+
+ result = 31 * result + (name != null ? name.hashCode() : 0);
+
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
index c101d04..2579023 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/query/h2/database/InlineIndexHelperTest.java
@@ -29,7 +29,7 @@ import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
import org.apache.ignite.internal.pagemem.PageIdAllocator;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
import org.apache.ignite.logger.java.JavaLogger;
import org.h2.result.SortOrder;
import org.h2.value.CompareMode;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
new file mode 100644
index 0000000..cb3cd1b
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.testsuites;
+
+import junit.framework.TestSuite;
+import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest;
+import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest;
+import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest;
+import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest;
+import org.apache.ignite.internal.processors.database.IgnitePersistentStoreSchemaLoadTest;
+
+/**
+ *
+ */
+public class IgnitePdsWithIndexingTestSuite extends TestSuite {
+ /**
+ * @return Test suite.
+ * @throws Exception Thrown in case of the failure.
+ */
+ public static TestSuite suite() throws Exception {
+ TestSuite suite = new TestSuite("Ignite Db Memory Leaks With Indexing Test Suite");
+
+ suite.addTestSuite(IgniteDbSingleNodeWithIndexingPutGetTest.class);
+ suite.addTestSuite(IgniteDbMultiNodeWithIndexingPutGetTest.class);
+ suite.addTestSuite(IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest.class);
+ suite.addTestSuite(IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest.class);
+ suite.addTestSuite(IgnitePersistentStoreSchemaLoadTest.class);
+
+ return suite;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/pom.xml
----------------------------------------------------------------------
diff --git a/modules/pds/pom.xml b/modules/pds/pom.xml
deleted file mode 100644
index 3bbccb9..0000000
--- a/modules/pds/pom.xml
+++ /dev/null
@@ -1,189 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!--
- POM file.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-parent</artifactId>
- <version>1</version>
- <relativePath>../../parent</relativePath>
- </parent>
-
- <artifactId>ignite-pds</artifactId>
- <version>2.1.0-SNAPSHOT</version>
-
- <properties>
- <common.io.version>2.4</common.io.version>
- <xstream.version>1.4.8</xstream.version>
- <guava.version>14.0.1</guava.version>
- <javassist.version>3.20.0-GA</javassist.version>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-core</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- <version>${common.io.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-spring</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.thoughtworks.xstream</groupId>
- <artifactId>xstream</artifactId>
- <version>${xstream.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-indexing</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-indexing</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-core</artifactId>
- <version>${project.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${guava.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.javassist</groupId>
- <artifactId>javassist</artifactId>
- <version>${javassist.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>1.3.2</version>
- <dependencies>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-tools</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
- <executions>
- <execution>
- <phase>process-classes</phase>
- <goals>
- <goal>java</goal>
- </goals>
- <configuration>
- <includePluginDependencies>true</includePluginDependencies>
- <mainClass>org.apache.ignite.tools.classgen.ClassesGenerator</mainClass>
- <arguments>
- <argument>${project.basedir}/target/classes</argument>
- <argument>
- <![CDATA[
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
- ]]>
- </argument>
- <argument>org.apache.ignite</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <version>2.2</version>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <version>2.8.2</version>
- <configuration>
- <skip>false</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/DbCheckpointListener.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/DbCheckpointListener.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/DbCheckpointListener.java
deleted file mode 100644
index 7f66790..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/DbCheckpointListener.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.util.Map;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.util.typedef.T2;
-
-/**
- *
- */
-public interface DbCheckpointListener {
- public interface Context {
- public boolean nextSnapshot();
-
- public Map<T2<Integer, Integer>, T2<Integer, Integer>> partitionStatMap();
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void onCheckpointBegin(Context context) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/FullPageIdIterableComparator.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/FullPageIdIterableComparator.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/FullPageIdIterableComparator.java
deleted file mode 100644
index 8e2afde..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/FullPageIdIterableComparator.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.io.Serializable;
-import java.util.Comparator;
-import org.apache.ignite.internal.util.typedef.T2;
-
-/**
- *
- */
-public class FullPageIdIterableComparator implements Comparator<T2<Integer, Integer>>, Serializable {
- /** */
- private static final long serialVersionUID = 0L;
-
- /** */
- public static final FullPageIdIterableComparator INSTANCE = new FullPageIdIterableComparator();
-
- /** {@inheritDoc} */
- @Override public int compare(T2<Integer, Integer> o1, T2<Integer, Integer> o2) {
- if (o1.get1() < o2.get1())
- return -1;
-
- if (o1.get1() > o2.get1())
- return 1;
-
- if (o1.get2() < o2.get2())
- return -1;
-
- if (o1.get2() > o2.get2())
- return 1;
-
- return 0;
- }
-}
-
[16/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheDatabaseSharedManager.java b/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheDatabaseSharedManager.java
deleted file mode 100755
index cae59a4..0000000
--- a/modules/pds/src/main/java/org/apache/ignite/internal/processors/cache/database/GridCacheDatabaseSharedManager.java
+++ /dev/null
@@ -1,3113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.ByteBuffer;
-import java.nio.ByteOrder;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.channels.OverlappingFileLockException;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.TreeMap;
-import java.util.UUID;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.CopyOnWriteArrayList;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import javax.management.InstanceNotFoundException;
-import javax.management.JMException;
-import javax.management.MBeanRegistrationException;
-import javax.management.ObjectName;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.PersistenceMetrics;
-import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.configuration.DataPageEvictionMode;
-import org.apache.ignite.configuration.IgniteConfiguration;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.configuration.PersistentStoreConfiguration;
-import org.apache.ignite.events.DiscoveryEvent;
-import org.apache.ignite.events.EventType;
-import org.apache.ignite.internal.GridKernalContext;
-import org.apache.ignite.internal.IgniteInternalFuture;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
-import org.apache.ignite.internal.NodeStoppingException;
-import org.apache.ignite.internal.mem.DirectMemoryProvider;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.snapshot.SnapshotOperation;
-import org.apache.ignite.internal.pagemem.store.IgnitePageStoreManager;
-import org.apache.ignite.internal.pagemem.store.PageStore;
-import org.apache.ignite.internal.pagemem.wal.StorageException;
-import org.apache.ignite.internal.pagemem.wal.WALIterator;
-import org.apache.ignite.internal.pagemem.wal.WALPointer;
-import org.apache.ignite.internal.pagemem.wal.record.CacheState;
-import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
-import org.apache.ignite.internal.pagemem.wal.record.DataEntry;
-import org.apache.ignite.internal.pagemem.wal.record.DataRecord;
-import org.apache.ignite.internal.pagemem.wal.record.MemoryRecoveryRecord;
-import org.apache.ignite.internal.pagemem.wal.record.PageSnapshot;
-import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionMetaStateRecord;
-import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.ClusterState;
-import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheProcessor;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.StoredCacheData;
-import org.apache.ignite.internal.processors.cache.database.file.FilePageStoreManager;
-import org.apache.ignite.internal.processors.cache.database.pagemem.CheckpointMetricsTracker;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryEx;
-import org.apache.ignite.internal.processors.cache.database.pagemem.PageMemoryImpl;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PagePartitionMetaIO;
-import org.apache.ignite.internal.processors.cache.database.wal.FileWALPointer;
-import org.apache.ignite.internal.processors.cache.database.wal.crc.PureJavaCrc32;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtPartitionState;
-import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture;
-import org.apache.ignite.internal.processors.port.GridPortRecord;
-import org.apache.ignite.internal.util.GridConcurrentHashSet;
-import org.apache.ignite.internal.util.GridMultiCollectionWrapper;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.future.CountDownFuture;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.lang.GridInClosure3X;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.CI1;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.P3;
-import org.apache.ignite.internal.util.typedef.T2;
-import org.apache.ignite.internal.util.typedef.internal.CU;
-import org.apache.ignite.internal.util.typedef.internal.LT;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.SB;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.internal.util.worker.GridWorker;
-import org.apache.ignite.lang.IgniteBiTuple;
-import org.apache.ignite.mxbean.PersistenceMetricsMXBean;
-import org.apache.ignite.thread.IgniteThread;
-import org.jetbrains.annotations.Nullable;
-
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
-import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
-
-/**
- *
- */
-@SuppressWarnings({"unchecked", "NonPrivateFieldAccessedInSynchronizedContext"})
-public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedManager {
- /** */
- public static final String IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC = "IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC";
-
- /** Skip sync. */
- private final boolean skipSync = IgniteSystemProperties.getBoolean(IGNITE_PDS_CHECKPOINT_TEST_SKIP_SYNC);
-
- /** */
- private boolean skipCrc = IgniteSystemProperties.getBoolean(IGNITE_PDS_SKIP_CRC, false);
-
- /** */
- private final int walRebalanceThreshold = IgniteSystemProperties.getInteger(
- IGNITE_PDS_WAL_REBALANCE_THRESHOLD, 500_000);
-
- /** Checkpoint lock hold count. */
- private static final ThreadLocal<Integer> CHECKPOINT_LOCK_HOLD_COUNT = new ThreadLocal<Integer>() {
- @Override protected Integer initialValue() {
- return 0;
- }
- };
-
- /** Assertion enabled. */
- private static final boolean ASSERTION_ENABLED = GridCacheDatabaseSharedManager.class.desiredAssertionStatus();
-
- /** Checkpoint file name pattern. */
- private static final Pattern CP_FILE_NAME_PATTERN = Pattern.compile("(\\d+)-(.*)-(START|END)\\.bin");
-
- /** */
- private static final FileFilter CP_FILE_FILTER = new FileFilter() {
- @Override public boolean accept(File f) {
- return CP_FILE_NAME_PATTERN.matcher(f.getName()).matches();
- }
- };
-
- /** */
- private static final Comparator<GridDhtLocalPartition> ASC_PART_COMPARATOR = new Comparator<GridDhtLocalPartition>() {
- @Override public int compare(GridDhtLocalPartition a, GridDhtLocalPartition b) {
- return Integer.compare(a.id(), b.id());
- }
- };
-
- /** */
- private static final Comparator<File> CP_TS_COMPARATOR = new Comparator<File>() {
- /** {@inheritDoc} */
- @Override public int compare(File o1, File o2) {
- Matcher m1 = CP_FILE_NAME_PATTERN.matcher(o1.getName());
- Matcher m2 = CP_FILE_NAME_PATTERN.matcher(o2.getName());
-
- boolean s1 = m1.matches();
- boolean s2 = m2.matches();
-
- assert s1 : "Failed to match CP file: " + o1.getAbsolutePath();
- assert s2 : "Failed to match CP file: " + o2.getAbsolutePath();
-
- long ts1 = Long.parseLong(m1.group(1));
- long ts2 = Long.parseLong(m2.group(1));
-
- int res = Long.compare(ts1, ts2);
-
- if (res == 0) {
- CheckpointEntryType type1 = CheckpointEntryType.valueOf(m1.group(3));
- CheckpointEntryType type2 = CheckpointEntryType.valueOf(m2.group(3));
-
- assert type1 != type2 : "o1=" + o1.getAbsolutePath() + ", o2=" + o2.getAbsolutePath();
-
- res = type1 == CheckpointEntryType.START ? -1 : 1;
- }
-
- return res;
- }
- };
-
- /** Checkpoint thread. Needs to be volatile because it is created in exchange worker. */
- private volatile Checkpointer checkpointer;
-
- /** For testing only. */
- private volatile boolean checkpointsEnabled = true;
-
- /** For testing only. */
- private volatile GridFutureAdapter<Void> enableChangeApplied;
-
- /** */
- private ReentrantReadWriteLock checkpointLock = new ReentrantReadWriteLock();
-
- /** */
- private long checkpointFreq;
-
- /** */
- private long checkpointPageBufSize;
-
- /** */
- private FilePageStoreManager storeMgr;
-
- /** */
- private File cpDir;
-
- /** */
- private volatile boolean printCheckpointStats = true;
-
- /** Database configuration. */
- private final PersistentStoreConfiguration persistenceCfg;
-
- /** */
- private final Collection<DbCheckpointListener> lsnrs = new CopyOnWriteArrayList<>();
-
- /** Checkpoint history. */
- private final CheckpointHistory checkpointHist = new CheckpointHistory();
-
- /** */
- private boolean stopping;
-
- /** Checkpoint runner thread pool. */
- private ExecutorService asyncRunner;
-
- /** Buffer for the checkpoint threads. */
- private ThreadLocal<ByteBuffer> threadBuf;
-
- /** */
- private final ConcurrentMap<Integer, IgniteInternalFuture> idxRebuildFuts = new ConcurrentHashMap<>();
-
- /** Lock holder. */
- private FileLockHolder fileLockHolder;
-
- /** Lock wait time. */
- private final int lockWaitTime;
-
- /** */
- private Map<Integer, Map<Integer, T2<Long, WALPointer>>> reservedForExchange;
-
- /** */
- private final ConcurrentMap<T2<Integer, Integer>, T2<Long, WALPointer>> reservedForPreloading = new ConcurrentHashMap<>();
-
- /** Snapshot manager. */
- private IgniteCacheSnapshotManager snapshotMgr;
-
- /** */
- private PersistenceMetricsImpl persStoreMetrics;
-
- /** */
- private ObjectName persistenceMetricsMbeanName;
-
- /**
- * @param ctx Kernal context.
- */
- public GridCacheDatabaseSharedManager(GridKernalContext ctx) {
- IgniteConfiguration cfg = ctx.config();
-
- persistenceCfg = cfg.getPersistentStoreConfiguration();
-
- assert persistenceCfg != null : "PageStore should not be created if persistence is disabled.";
-
- checkpointFreq = persistenceCfg.getCheckpointingFrequency();
-
- lockWaitTime = persistenceCfg.getLockWaitTime();
-
- final int pageSize = cfg.getMemoryConfiguration().getPageSize();
-
- threadBuf = new ThreadLocal<ByteBuffer>() {
- /** {@inheritDoc} */
- @Override protected ByteBuffer initialValue() {
- ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(pageSize);
-
- tmpWriteBuf.order(ByteOrder.nativeOrder());
-
- return tmpWriteBuf;
- }
- };
-
- persStoreMetrics = new PersistenceMetricsImpl(
- persistenceCfg.isMetricsEnabled(),
- persistenceCfg.getRateTimeInterval(),
- persistenceCfg.getSubIntervals()
- );
- }
-
- /**
- *
- */
- public Checkpointer getCheckpointer() {
- return checkpointer;
- }
-
- /**
- * For test use only.
- */
- public IgniteInternalFuture<Void> enableCheckpoints(boolean enable) {
- GridFutureAdapter<Void> fut = new GridFutureAdapter<>();
-
- enableChangeApplied = fut;
-
- checkpointsEnabled = enable;
-
- wakeupForCheckpoint("enableCheckpoints()");
-
- return fut;
- }
-
- /** {@inheritDoc} */
- @Override protected void start0() throws IgniteCheckedException {
- snapshotMgr = cctx.snapshot();
-
- assert !cctx.kernalContext().state().active() : "Cluster with persistent must starting as inactive.";
-
- if (!cctx.kernalContext().clientNode()) {
- IgnitePageStoreManager store = cctx.pageStore();
-
- assert store instanceof FilePageStoreManager : "Invalid page store manager was created: " + store;
-
- storeMgr = (FilePageStoreManager)store;
-
- cpDir = Paths.get(storeMgr.workDir().getAbsolutePath(), "cp").toFile();
-
- if (!U.mkdirs(cpDir))
- throw new IgniteCheckedException("Could not create directory for checkpoint metadata: " + cpDir);
-
- fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log);
-
- persStoreMetrics.wal(cctx.wal());
-
- try {
- persistenceMetricsMbeanName = U.registerMBean(
- cctx.kernalContext().config().getMBeanServer(),
- cctx.kernalContext().igniteInstanceName(),
- "Persistent Store",
- "PersistenceMetrics",
- persStoreMetrics,
- PersistenceMetricsMXBean.class);
- }
- catch (JMException e) {
- throw new IgniteCheckedException("Failed to register persistence metrics MBean", e);
- }
- }
- }
-
- /**
- *
- */
- @Override public void initDataBase() throws IgniteCheckedException {
- Long cpBufSize = persistenceCfg.getCheckpointingPageBufferSize();
-
- if (persistenceCfg.getCheckpointingThreads() > 1)
- asyncRunner = new ThreadPoolExecutor(
- persistenceCfg.getCheckpointingThreads(),
- persistenceCfg.getCheckpointingThreads(),
- 30L,
- TimeUnit.SECONDS,
- new LinkedBlockingQueue<Runnable>()
- );
-
- // Intentionally use identity comparison to check if configuration default has changed.
- //noinspection NumberEquality
- if (cpBufSize == PersistentStoreConfiguration.DFLT_CHECKPOINTING_PAGE_BUFFER_SIZE) {
- MemoryConfiguration memCfg = cctx.kernalContext().config().getMemoryConfiguration();
-
- assert memCfg != null;
-
- long totalSize = memCfg.getSystemCacheMaxSize();
-
- if (memCfg.getMemoryPolicies() == null)
- totalSize += MemoryConfiguration.DFLT_MEMORY_POLICY_MAX_SIZE;
- else {
- for (MemoryPolicyConfiguration memPlc : memCfg.getMemoryPolicies()) {
- if (Long.MAX_VALUE - memPlc.getMaxSize() > totalSize)
- totalSize += memPlc.getMaxSize();
- else {
- totalSize = Long.MAX_VALUE;
-
- break;
- }
- }
-
- assert totalSize > 0;
- }
-
- // Limit the checkpoint page buffer size by 2GB.
- long dfltSize = 2 * 1024L * 1024L * 1024L;
-
- long adjusted = Math.min(totalSize / 4, dfltSize);
-
- if (cpBufSize < adjusted) {
- U.quietAndInfo(log,
- "Default checkpoint page buffer size is too small, setting to an adjusted value: "
- + U.readableSize(adjusted, false)
- );
-
- cpBufSize = adjusted;
- }
- }
-
- checkpointPageBufSize = cpBufSize;
-
- super.start0();
- }
-
- /** {@inheritDoc} */
- @Override protected void initPageMemoryDataStructures(MemoryConfiguration dbCfg) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override protected void onKernalStart0(boolean reconnect) throws IgniteCheckedException {
- if (reconnect || cctx.kernalContext().clientNode() || !cctx.kernalContext().state().active())
- return;
-
- if (persistenceEnabled() && cctx.kernalContext().state().active())
- initDataBase();
-
- GridCacheProcessor cachePrc = cctx.kernalContext().cache();
-
- // Todo before join local node.
-
- Collection<String> cacheNames = new HashSet<>();
-
- // TODO IGNITE-5075 group descriptors.
- for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration()) {
- if (CU.isSystemCache(ccfg.getName())) {
- storeMgr.initializeForCache(
- cctx.cache().cacheDescriptors().get(ccfg.getName()).groupDescriptor(), new StoredCacheData(ccfg));
-
- cacheNames.add(ccfg.getName());
- }
- }
-
- for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration())
- if (!CU.isSystemCache(ccfg.getName())) {
- DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptors().get(ccfg.getName());
-
- if (cacheDesc != null)
- storeMgr.initializeForCache(cacheDesc.groupDescriptor(), new StoredCacheData(ccfg));
-
- cacheNames.add(ccfg.getName());
- }
-
- for (StoredCacheData cacheData : cctx.pageStore().readCacheConfigurations().values()) {
- if (!cacheNames.contains(cacheData.config().getName()))
- storeMgr.initializeForCache(
- cctx.cache().cacheDescriptors().get(cacheData.config().getName()).groupDescriptor(), cacheData);
- }
-
- readCheckpointAndRestoreMemory();
- }
-
-
- /** {@inheritDoc} */
- @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
- snapshotMgr = cctx.snapshot();
-
- if (cctx.localNode().isClient())
- return;
-
- initDataBase();
-
- if (log.isDebugEnabled())
- log.debug("Activate database manager [id=" + cctx.localNodeId() +
- " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
-
- GridCacheProcessor cachePrc = cctx.kernalContext().cache();
-
- // Todo join local info.
-
- Collection<String> cacheNames = new HashSet<>();
-
- // TODO IGNITE-5075 group descriptors.
- for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration()) {
- if (CU.isSystemCache(ccfg.getName())) {
- storeMgr.initializeForCache(
- cctx.cache().cacheDescriptors().get(ccfg.getName()).groupDescriptor(), new StoredCacheData(ccfg));
-
- cacheNames.add(ccfg.getName());
- }
- }
-
- for (CacheConfiguration ccfg : cctx.kernalContext().config().getCacheConfiguration())
- if (!CU.isSystemCache(ccfg.getName())) {
- DynamicCacheDescriptor cacheDesc = cctx.cache().cacheDescriptors().get(ccfg.getName());
-
- if (cacheDesc != null)
- storeMgr.initializeForCache(cacheDesc.groupDescriptor(), new StoredCacheData(ccfg));
-
- cacheNames.add(ccfg.getName());
- }
-
- for (StoredCacheData cacheData : cctx.pageStore().readCacheConfigurations().values()) {
- if (!cacheNames.contains(cacheData.config().getName()))
- storeMgr.initializeForCache(
- cctx.cache().cacheDescriptors().get(cacheData.config().getName()).groupDescriptor(), cacheData);
- }
-
- readCheckpointAndRestoreMemory();
-
- if (log.isDebugEnabled())
- log.debug("Restore state after activation [nodeId=" + cctx.localNodeId() + " ]");
- }
-
- /** {@inheritDoc} */
- @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
- stop0(true);
-
- if (log.isDebugEnabled())
- log.debug("DeActivate database manager [id=" + cctx.localNodeId() +
- " topVer=" + cctx.discovery().topologyVersionEx() + " ]");
-
- onKernalStop0(false);
-
- /* Must be here, because after deactivate we can invoke activate and file lock must be already configured */
- stopping = false;
-
- fileLockHolder = new FileLockHolder(storeMgr.workDir().getPath(), cctx.kernalContext(), log);
- }
-
- /**
- *
- */
- private void readCheckpointAndRestoreMemory() throws IgniteCheckedException {
- checkpointReadLock();
-
- try {
- CheckpointStatus status = readCheckpointStatus();
-
- // First, bring memory to the last consistent checkpoint state if needed.
- // This method should return a pointer to the last valid record in the WAL.
- WALPointer restore = restoreMemory(status);
-
- cctx.wal().resumeLogging(restore);
-
- cctx.wal().log(new MemoryRecoveryRecord(U.currentTimeMillis()));
- }
- catch (StorageException e) {
- throw new IgniteCheckedException(e);
- }
- finally {
- checkpointReadUnlock();
- }
- }
-
- /** {@inheritDoc} */
- @Override public void lock() throws IgniteCheckedException {
- if (log.isDebugEnabled())
- log.debug("Try to capture file lock [nodeId=" +
- cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
- fileLockHolder.tryLock(lockWaitTime);
- }
-
- /** {@inheritDoc} */
- @Override public void unLock() {
- if (log.isDebugEnabled())
- log.debug("Release file lock [nodeId=" +
- cctx.localNodeId() + " path=" + fileLockHolder.lockPath() + "]");
-
- fileLockHolder.release();
- }
-
- /** {@inheritDoc} */
- @Override public void onCacheStop(GridCacheContext cctx) {
- snapshotMgr.onCacheStop(cctx);
- }
-
- /** {@inheritDoc} */
- @Override protected void onKernalStop0(boolean cancel) {
- checkpointLock.writeLock().lock();
-
- try {
- stopping = true;
- }
- finally {
- checkpointLock.writeLock().unlock();
- }
-
- snapshotMgr.onKernalStop(cancel);
-
- shutdownCheckpointer(cancel);
-
- lsnrs.clear();
-
- super.onKernalStop0(cancel);
-
- if (!cctx.kernalContext().clientNode()) {
- unLock();
-
- fileLockHolder.close();
- }
-
- if (persistenceMetricsMbeanName != null) {
- try {
- cctx.kernalContext().config().getMBeanServer().unregisterMBean(persistenceMetricsMbeanName);
- }
- catch (InstanceNotFoundException ignore) {
- // No-op, nothing to unregister.
- }
- catch (MBeanRegistrationException e) {
- U.error(log, "Failed to unregister persistence metrics MBean (will continue stop routine)", e);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override protected void stop0(boolean cancel) {
- super.stop0(cancel);
-
- snapshotMgr.stop(cancel);
- }
-
- /** */
- private long[] calculateFragmentSizes(int concLvl, long cacheSize) {
- if (concLvl < 2)
- concLvl = Runtime.getRuntime().availableProcessors();
-
- long fragmentSize = cacheSize / concLvl;
-
- if (fragmentSize < 1024 * 1024)
- fragmentSize = 1024 * 1024;
-
- long[] sizes = new long[concLvl + 1];
-
- for (int i = 0; i < concLvl; i++)
- sizes[i] = fragmentSize;
-
- sizes[concLvl] = checkpointPageBufSize;
-
- return sizes;
- }
-
- /** {@inheritDoc} */
- @Override protected PageMemory createPageMemory(
- DirectMemoryProvider memProvider,
- MemoryConfiguration memCfg,
- MemoryPolicyConfiguration plcCfg,
- MemoryMetricsImpl memMetrics
- ) {
- memMetrics.persistenceEnabled(true);
-
- PageMemoryImpl pageMem = new PageMemoryImpl(
- memProvider,
- calculateFragmentSizes(
- memCfg.getConcurrencyLevel(),
- plcCfg.getMaxSize()
- ),
- cctx,
- memCfg.getPageSize(),
- new GridInClosure3X<FullPageId, ByteBuffer, Integer>() {
- @Override public void applyx(
- FullPageId fullId,
- ByteBuffer pageBuf,
- Integer tag
- ) throws IgniteCheckedException {
- storeMgr.write(fullId.cacheId(), fullId.pageId(), pageBuf, tag);
-
- snapshotMgr.flushDirtyPageHandler(fullId, pageBuf, tag);
- }
- },
- new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
- @Override public void applyx(
- Long page,
- FullPageId fullId,
- PageMemoryEx pageMem
- ) throws IgniteCheckedException {
- snapshotMgr.onChangeTrackerPage(page, fullId, pageMem);
- }
- },
- this,
- memMetrics
- );
-
- memMetrics.pageMemory(pageMem);
-
- return pageMem;
- }
-
- /** {@inheritDoc} */
- @Override protected void checkPolicyEvictionProperties(MemoryPolicyConfiguration plcCfg, MemoryConfiguration dbCfg)
- throws IgniteCheckedException {
- if (plcCfg.getPageEvictionMode() != DataPageEvictionMode.DISABLED)
- throw new IgniteCheckedException("Page eviction is not compatible with persistence: " + plcCfg.getName());
- }
-
- /**
- * @param cancel Cancel flag.
- */
- @SuppressWarnings("unused")
- private void shutdownCheckpointer(boolean cancel) {
- Checkpointer cp = checkpointer;
-
- if (cp != null) {
- if (cancel)
- cp.shutdownNow();
- else
- cp.cancel();
-
- try {
- U.join(cp);
-
- checkpointer = null;
- }
- catch (IgniteInterruptedCheckedException ignore) {
- U.warn(log, "Was interrupted while waiting for checkpointer shutdown, " +
- "will not wait for checkpoint to finish.");
-
- cp.shutdownNow();
-
- while (true) {
- try {
- U.join(cp);
-
- checkpointer = null;
-
- cp.scheduledCp.cpFinishFut.onDone(
- new NodeStoppingException("Checkpointer is stopped during node stop."));
-
- break;
- }
- catch (IgniteInterruptedCheckedException ignored) {
- //Ignore
- }
- }
-
- Thread.currentThread().interrupt();
- }
- }
-
- if (asyncRunner != null) {
- asyncRunner.shutdownNow();
-
- try {
- asyncRunner.awaitTermination(2, TimeUnit.MINUTES);
- }
- catch (InterruptedException ignore) {
- Thread.currentThread().interrupt();
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public void beforeExchange(GridDhtPartitionsExchangeFuture fut) throws IgniteCheckedException {
- DiscoveryEvent discoEvt = fut.discoveryEvent();
-
- boolean joinEvt = discoEvt.type() == EventType.EVT_NODE_JOINED;
-
- boolean locNode = discoEvt.eventNode().isLocal();
-
- boolean isSrvNode = !cctx.kernalContext().clientNode();
-
- boolean clusterStatusActive = cctx.kernalContext().state().active();
-
- boolean clusterInTransitionStateToActive = fut.newClusterState() == ClusterState.ACTIVE;
-
- // Before local node join event.
- if (clusterInTransitionStateToActive ||
- (joinEvt && locNode && isSrvNode && clusterStatusActive))
- restoreState();
-
- if (cctx.kernalContext().query().moduleEnabled()) {
- for (GridCacheContext cacheCtx : (Collection<GridCacheContext>)cctx.cacheContexts()) {
- if (cacheCtx.startTopologyVersion().equals(fut.topologyVersion()) &&
- !cctx.pageStore().hasIndexStore(cacheCtx.groupId()) && cacheCtx.affinityNode()) {
- final int cacheId = cacheCtx.cacheId();
-
- final IgniteInternalFuture<?> rebuildFut = cctx.kernalContext().query()
- .rebuildIndexesFromHash(Collections.singletonList(cacheCtx.cacheId()));
-
- idxRebuildFuts.put(cacheId, rebuildFut);
-
- rebuildFut.listen(new CI1<IgniteInternalFuture>() {
- @Override public void apply(IgniteInternalFuture igniteInternalFut) {
- idxRebuildFuts.remove(cacheId, rebuildFut);
- }
- });
- }
- }
- }
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public IgniteInternalFuture indexRebuildFuture(int cacheId) {
- return idxRebuildFuts.get(cacheId);
- }
-
- /** {@inheritDoc} */
- @Override public boolean persistenceEnabled() {
- return true;
- }
-
- /** {@inheritDoc} */
- @Override public void onCacheGroupsStopped(
- Collection<IgniteBiTuple<CacheGroupContext, Boolean>> stoppedGrps) {
- try {
- waitForCheckpoint("caches stop");
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to wait for checkpoint finish during cache stop.", e);
- }
-
- Map<PageMemoryEx, Collection<Integer>> destroyed = new HashMap<>();
-
- for (IgniteBiTuple<CacheGroupContext, Boolean> tup : stoppedGrps) {
- PageMemoryEx pageMem = (PageMemoryEx)tup.get1().memoryPolicy().pageMemory();
-
- Collection<Integer> grpIds = destroyed.get(pageMem);
-
- if (grpIds == null) {
- grpIds = new HashSet<>();
-
- destroyed.put(pageMem, grpIds);
- }
-
- grpIds.add(tup.get1().groupId());
-
- pageMem.onCacheGroupDestroyed(tup.get1().groupId());
- }
-
- Collection<IgniteInternalFuture<Void>> clearFuts = new ArrayList<>(destroyed.size());
-
- for (Map.Entry<PageMemoryEx, Collection<Integer>> entry : destroyed.entrySet()) {
- final Collection<Integer> grpIds = entry.getValue();
-
- clearFuts.add(entry.getKey().clearAsync(new P3<Integer, Long, Integer>() {
- @Override public boolean apply(Integer grpId, Long pageId, Integer tag) {
- return grpIds.contains(grpId);
- }
- }, false));
- }
- for (IgniteInternalFuture<Void> clearFut : clearFuts) {
- try {
- clearFut.get();
- }
- catch (IgniteCheckedException e) {
- log.error("Failed to clear page memory", e);
- }
- }
-
- if (cctx.pageStore() != null) {
- for (IgniteBiTuple<CacheGroupContext, Boolean> tup : stoppedGrps) {
- CacheGroupContext grp = tup.get1();
-
- if (grp.affinityNode()) {
- try {
- cctx.pageStore().shutdownForCacheGroup(grp, tup.get2());
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Failed to gracefully clean page store resources for destroyed cache " +
- "[cache=" + grp.cacheOrGroupName() + "]", e);
- }
- }
- }
- }
- }
-
- /**
- * Gets the checkpoint read lock. While this lock is held, checkpoint thread will not acquiSnapshotWorkerre memory state.
- */
- @SuppressWarnings("LockAcquiredButNotSafelyReleased")
- @Override public void checkpointReadLock() {
- if (checkpointLock.writeLock().isHeldByCurrentThread())
- return;
-
- for (; ; ) {
- checkpointLock.readLock().lock();
-
- if (stopping) {
- checkpointLock.readLock().unlock();
-
- throw new RuntimeException("Failed to perform cache update: node is stopping.");
- }
-
- if (safeToUpdatePageMemories() || checkpointLock.getReadHoldCount() > 1)
- break;
- else {
- checkpointLock.readLock().unlock();
-
- try {
- checkpointer.wakeupForCheckpoint(0, "too many dirty pages").cpBeginFut.get();
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException("Failed to wait for checkpoint begin.", e);
- }
- }
- }
-
- if (ASSERTION_ENABLED)
- CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() + 1);
- }
-
- /** {@inheritDoc} */
- @Override public boolean checkpointLockIsHeldByThread() {
- return !ASSERTION_ENABLED ||
- checkpointLock.isWriteLockedByCurrentThread() ||
- CHECKPOINT_LOCK_HOLD_COUNT.get() > 0;
- }
-
- /**
- * @return {@code true} if all PageMemory instances are safe to update.
- */
- private boolean safeToUpdatePageMemories() {
- Collection<MemoryPolicy> memPlcs = context().database().memoryPolicies();
-
- if (memPlcs == null)
- return true;
-
- for (MemoryPolicy memPlc : memPlcs) {
- PageMemoryEx pageMemEx = (PageMemoryEx) memPlc.pageMemory();
-
- if (!pageMemEx.safeToUpdate())
- return false;
- }
-
- return true;
- }
-
- /**
- * Releases the checkpoint read lock.
- */
- @Override public void checkpointReadUnlock() {
- if (checkpointLock.writeLock().isHeldByCurrentThread())
- return;
-
- checkpointLock.readLock().unlock();
-
- if (checkpointer != null) {
- Collection<MemoryPolicy> memPlcs = context().database().memoryPolicies();
-
- if (memPlcs != null) {
- for (MemoryPolicy memPlc : memPlcs) {
- PageMemoryEx mem = (PageMemoryEx)memPlc.pageMemory();
-
- if (mem != null && !mem.safeToUpdate()) {
- checkpointer.wakeupForCheckpoint(0, "too many dirty pages");
-
- break;
- }
- }
- }
- }
-
- if (ASSERTION_ENABLED)
- CHECKPOINT_LOCK_HOLD_COUNT.set(CHECKPOINT_LOCK_HOLD_COUNT.get() - 1);
- }
-
- /**
- * @throws IgniteCheckedException If failed to restore database status from WAL.
- */
- private void restoreState() throws IgniteCheckedException {
- try {
- CheckpointStatus status = readCheckpointStatus();
-
- checkpointReadLock();
-
- try {
- applyLastUpdates(status);
- }
- finally {
- checkpointReadUnlock();
- }
-
- snapshotMgr.restoreState();
-
- checkpointer = new Checkpointer(cctx.igniteInstanceName(), "db-checkpoint-thread", log);
-
- new IgniteThread(cctx.igniteInstanceName(), "db-checkpoint-thread", checkpointer).start();
- }
- catch (StorageException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public synchronized Map<Integer, Map<Integer, Long>> reserveHistoryForExchange() {
- assert reservedForExchange == null : reservedForExchange;
-
- reservedForExchange = new HashMap<>();
-
- for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
- if (grp.isLocal())
- continue;
-
- for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions()) {
- if (part.state() != GridDhtPartitionState.OWNING || part.dataStore().fullSize() <= walRebalanceThreshold)
- continue;
-
- CheckpointEntry cpEntry = searchCheckpointEntry(grp.groupId(), part.id(), null);
-
- try {
- if (cpEntry != null && cctx.wal().reserve(cpEntry.cpMark)) {
- Map<Integer, T2<Long, WALPointer>> cacheMap = reservedForExchange.get(grp.groupId());
-
- if (cacheMap == null) {
- cacheMap = new HashMap<>();
-
- reservedForExchange.put(grp.groupId(), cacheMap);
- }
-
- cacheMap.put(part.id(), new T2<>(cpEntry.partitionCounter(grp.groupId(), part.id()), cpEntry.cpMark));
- }
- }
- catch (IgniteCheckedException ex) {
- U.error(log, "Error while trying to reserve history", ex);
- }
- }
- }
-
- Map<Integer, Map<Integer, Long>> resMap = new HashMap<>();
-
- for (Map.Entry<Integer, Map<Integer, T2<Long, WALPointer>>> e : reservedForExchange.entrySet()) {
- Map<Integer, Long> cacheMap = new HashMap<>();
-
- for (Map.Entry<Integer, T2<Long, WALPointer>> e0 : e.getValue().entrySet())
- cacheMap.put(e0.getKey(), e0.getValue().get1());
-
- resMap.put(e.getKey(), cacheMap);
- }
-
- return resMap;
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void releaseHistoryForExchange() {
- if (reservedForExchange == null)
- return;
-
- for (Map.Entry<Integer, Map<Integer, T2<Long, WALPointer>>> e : reservedForExchange.entrySet()) {
- for (Map.Entry<Integer, T2<Long, WALPointer>> e0 : e.getValue().entrySet()) {
- try {
- cctx.wal().release(e0.getValue().get2());
- }
- catch (IgniteCheckedException ex) {
- U.error(log, "Could not release history lock", ex);
- }
- }
- }
-
- reservedForExchange = null;
- }
-
- /** {@inheritDoc} */
- @Override public boolean reserveHistoryForPreloading(int grpId, int partId, long cntr) {
- CheckpointEntry cpEntry = searchCheckpointEntry(grpId, partId, cntr);
-
- if (cpEntry == null)
- return false;
-
- WALPointer ptr = cpEntry.cpMark;
-
- if (ptr == null)
- return false;
-
- boolean reserved;
-
- try {
- reserved = cctx.wal().reserve(ptr);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Error while trying to reserve history", e);
-
- reserved = false;
- }
-
- if (reserved)
- reservedForPreloading.put(new T2<>(grpId, partId), new T2<>(cntr, ptr));
-
- return reserved;
- }
-
- /** {@inheritDoc} */
- @Override public void releaseHistoryForPreloading() {
- for (Map.Entry<T2<Integer, Integer>, T2<Long, WALPointer>> e : reservedForPreloading.entrySet()) {
- try {
- cctx.wal().release(e.getValue().get2());
- }
- catch (IgniteCheckedException ex) {
- U.error(log, "Could not release WAL reservation", ex);
-
- throw new IgniteException(ex);
- }
- }
-
- reservedForPreloading.clear();
- }
-
- /**
- * For debugging only. TODO: remove.
- *
- */
- public Map<T2<Integer, Integer>, T2<Long, WALPointer>> reservedForPreloading() {
- return reservedForPreloading;
- }
-
- /**
- *
- */
- @Nullable @Override public IgniteInternalFuture wakeupForCheckpoint(String reason) {
- Checkpointer cp = checkpointer;
-
- if (cp != null)
- return cp.wakeupForCheckpoint(0, reason).cpBeginFut;
-
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void waitForCheckpoint(String reason) throws IgniteCheckedException {
- Checkpointer cp = checkpointer;
-
- if (cp == null)
- return;
-
- CheckpointProgressSnapshot progSnapshot = cp.wakeupForCheckpoint(0, reason);
-
- IgniteInternalFuture fut1 = progSnapshot.cpFinishFut;
-
- fut1.get();
-
- if (!progSnapshot.started)
- return;
-
- IgniteInternalFuture fut2 = cp.wakeupForCheckpoint(0, reason).cpFinishFut;
-
- assert fut1 != fut2;
-
- fut2.get();
- }
-
- /**
- * Tries to search for a WAL pointer for the given partition counter start.
- *
- * @param grpId Cache group ID.
- * @param part Partition ID.
- * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
- * @return Checkpoint entry or {@code null} if failed to search.
- */
- @Nullable public WALPointer searchPartitionCounter(int grpId, int part, @Nullable Long partCntrSince) {
- CheckpointEntry entry = searchCheckpointEntry(grpId, part, partCntrSince);
-
- if (entry == null)
- return null;
-
- return entry.cpMark;
- }
-
- /**
- * Tries to search for a WAL pointer for the given partition counter start.
- *
- * @param grpId Cache group ID.
- * @param part Partition ID.
- * @param partCntrSince Partition counter or {@code null} to search for minimal counter.
- * @return Checkpoint entry or {@code null} if failed to search.
- */
- @Nullable private CheckpointEntry searchCheckpointEntry(int grpId, int part, @Nullable Long partCntrSince) {
- boolean hasGap = false;
- CheckpointEntry first = null;
-
- for (Long cpTs : checkpointHist.checkpoints()) {
- try {
- CheckpointEntry entry = checkpointHist.entry(cpTs);
-
- Long foundCntr = entry.partitionCounter(grpId, part);
-
- if (foundCntr != null) {
- if (partCntrSince == null) {
- if (hasGap) {
- first = entry;
-
- hasGap = false;
- }
-
- if (first == null)
- first = entry;
- }
- else if (foundCntr <= partCntrSince) {
- first = entry;
-
- hasGap = false;
- }
- else
- return hasGap ? null : first;
- }
- else
- hasGap = true;
- }
- catch (IgniteCheckedException ignore) {
- // Treat exception the same way as a gap.
- hasGap = true;
- }
- }
-
- return hasGap ? null : first;
- }
-
- /**
- * @return Checkpoint history. For tests only.
- */
- public CheckpointHistory checkpointHistory() {
- return checkpointHist;
- }
-
- /**
- * @return Checkpoint directory.
- */
- public File checkpointDirectory() {
- return cpDir;
- }
-
- /**
- * @param lsnr Listener.
- */
- public void addCheckpointListener(DbCheckpointListener lsnr) {
- lsnrs.add(lsnr);
- }
-
- /**
- * @param lsnr Listener.
- */
- public void removeCheckpointListener(DbCheckpointListener lsnr) {
- lsnrs.remove(lsnr);
- }
-
- /**
- * @return Read checkpoint status.
- * @throws IgniteCheckedException If failed to read checkpoint status page.
- */
- @SuppressWarnings("TooBroadScope")
- private CheckpointStatus readCheckpointStatus() throws IgniteCheckedException {
- long lastStartTs = 0;
- long lastEndTs = 0;
-
- UUID startId = CheckpointStatus.NULL_UUID;
- UUID endId = CheckpointStatus.NULL_UUID;
-
- File startFile = null;
- File endFile = null;
-
- WALPointer startPtr = CheckpointStatus.NULL_PTR;
- WALPointer endPtr = CheckpointStatus.NULL_PTR;
-
- File dir = cpDir;
-
- if (!dir.exists()) {
- // TODO: remove excessive logging after GG-12116 fix.
- File[] files = dir.listFiles();
-
- if (files != null && files.length > 0) {
- log.warning("Read checkpoint status: cpDir.exists() is false, cpDir.listFiles() is: " +
- Arrays.toString(files));
- }
-
- if (Files.exists(dir.toPath()))
- log.warning("Read checkpoint status: cpDir.exists() is false, Files.exists(cpDir) is true.");
-
- log.info("Read checkpoint status: checkpoint directory is not found.");
-
- return new CheckpointStatus(0, startId, startPtr, endId, endPtr);
- }
-
- File[] files = dir.listFiles();
-
- for (File file : files) {
- Matcher matcher = CP_FILE_NAME_PATTERN.matcher(file.getName());
-
- if (matcher.matches()) {
- long ts = Long.parseLong(matcher.group(1));
- UUID id = UUID.fromString(matcher.group(2));
- CheckpointEntryType type = CheckpointEntryType.valueOf(matcher.group(3));
-
- if (type == CheckpointEntryType.START && ts > lastStartTs) {
- lastStartTs = ts;
- startId = id;
- startFile = file;
- }
- else if (type == CheckpointEntryType.END && ts > lastEndTs) {
- lastEndTs = ts;
- endId = id;
- endFile = file;
- }
- }
- }
-
- ByteBuffer buf = ByteBuffer.allocate(20);
- buf.order(ByteOrder.nativeOrder());
-
- if (startFile != null)
- startPtr = readPointer(startFile, buf);
-
- if (endFile != null)
- endPtr = readPointer(endFile, buf);
-
- // TODO: remove excessive logging after GG-12116 fix.
- log.info("Read checkpoint status: start marker = " + startFile + ", end marker = " + endFile);
-
- return new CheckpointStatus(lastStartTs, startId, startPtr, endId, endPtr);
- }
-
- /**
- * @param cpMarkerFile Checkpoint mark file.
- * @return WAL pointer.
- * @throws IgniteCheckedException If failed to read mark file.
- */
- private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteCheckedException {
- buf.position(0);
-
- try (FileChannel ch = FileChannel.open(cpMarkerFile.toPath(), StandardOpenOption.READ)) {
- ch.read(buf);
-
- buf.flip();
-
- return new FileWALPointer(buf.getLong(), buf.getInt(), buf.getInt());
- }
- catch (IOException e) {
- throw new IgniteCheckedException("Failed to read checkpoint pointer from marker file: " +
- cpMarkerFile.getAbsolutePath(), e);
- }
- }
-
- /**
- * @param status Checkpoint status.
- */
- private WALPointer restoreMemory(CheckpointStatus status) throws IgniteCheckedException {
- if (log.isInfoEnabled())
- log.info("Checking memory state [lastValidPos=" + status.endPtr + ", lastMarked="
- + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
-
- boolean apply = status.needRestoreMemory();
-
- if (apply) {
- U.quietAndWarn(log, "Ignite node crashed in the middle of checkpoint. Will restore memory state and " +
- "enforce checkpoint on node start.");
-
- cctx.pageStore().beginRecover();
- }
-
- long start = U.currentTimeMillis();
- int applied = 0;
- WALPointer lastRead = null;
-
- try (WALIterator it = cctx.wal().replay(status.endPtr)) {
- while (it.hasNextX()) {
- IgniteBiTuple<WALPointer, WALRecord> tup = it.nextX();
-
- WALRecord rec = tup.get2();
-
- lastRead = tup.get1();
-
- switch (rec.type()) {
- case CHECKPOINT_RECORD:
- CheckpointRecord cpRec = (CheckpointRecord)rec;
-
- // We roll memory up until we find a checkpoint start record registered in the status.
- if (F.eq(cpRec.checkpointId(), status.cpStartId)) {
- log.info("Found last checkpoint marker [cpId=" + cpRec.checkpointId() +
- ", pos=" + tup.get1() + ']');
-
- apply = false;
- }
- else if (!F.eq(cpRec.checkpointId(), status.cpEndId))
- U.warn(log, "Found unexpected checkpoint marker, skipping [cpId=" + cpRec.checkpointId() +
- ", expCpId=" + status.cpStartId + ", pos=" + tup.get1() + ']');
-
- break;
-
- case PAGE_RECORD:
- if (apply) {
- PageSnapshot pageRec = (PageSnapshot)rec;
-
- // Here we do not require tag check because we may be applying memory changes after
- // several repetitive restarts and the same pages may have changed several times.
- int cacheId = pageRec.fullPageId().cacheId();
- long pageId = pageRec.fullPageId().pageId();
-
- PageMemoryEx pageMem = getPageMemoryForCacheGroup(cacheId);
-
- long page = pageMem.acquirePage(cacheId, pageId, true);
-
- try {
- long pageAddr = pageMem.writeLock(cacheId, pageId, page);
-
- try {
- PageUtils.putBytes(pageAddr, 0, pageRec.pageData());
- }
- finally {
- pageMem.writeUnlock(cacheId, pageId, page, null, true, true);
- }
- }
- finally {
- pageMem.releasePage(cacheId, pageId, page);
- }
-
- applied++;
- }
-
- break;
-
- case PARTITION_DESTROY:
- if (apply) {
- PartitionDestroyRecord destroyRec = (PartitionDestroyRecord)rec;
-
- final int cId = destroyRec.cacheId();
- final int pId = destroyRec.partitionId();
-
- PageMemoryEx pageMem = getPageMemoryForCacheGroup(cId);
-
- pageMem.clearAsync(new P3<Integer, Long, Integer>() {
- @Override public boolean apply(Integer cacheId, Long pageId, Integer tag) {
- return cacheId == cId && PageIdUtils.partId(pageId) == pId;
- }
- }, true).get();
- }
-
- break;
-
- default:
- if (apply && rec instanceof PageDeltaRecord) {
- PageDeltaRecord r = (PageDeltaRecord)rec;
-
- int cacheId = r.cacheId();
- long pageId = r.pageId();
-
- PageMemoryEx pageMem = getPageMemoryForCacheGroup(cacheId);
-
- // Here we do not require tag check because we may be applying memory changes after
- // several repetitive restarts and the same pages may have changed several times.
- long page = pageMem.acquirePage(cacheId, pageId, true);
-
- try {
- long pageAddr = pageMem.writeLock(cacheId, pageId, page);
-
- try {
- r.applyDelta(pageMem, pageAddr);
- }
- finally {
- pageMem.writeUnlock(cacheId, pageId, page, null, true, true);
- }
- }
- finally {
- pageMem.releasePage(cacheId, pageId, page);
- }
-
- applied++;
- }
- }
- }
- }
-
- if (status.needRestoreMemory()) {
- if (apply)
- throw new IgniteCheckedException("Failed to restore memory state (checkpoint marker is present " +
- "on disk, but checkpoint record is missed in WAL) " +
- "[cpStatus=" + status + ", lastRead=" + lastRead + "]");
-
- log.info("Finished applying memory changes [changesApplied=" + applied +
- ", time=" + (U.currentTimeMillis() - start) + "ms]");
-
- if (applied > 0)
- finalizeCheckpointOnRecovery(status.cpStartTs, status.cpStartId, status.startPtr);
- }
-
- checkpointHist.loadHistory(cpDir);
-
- return lastRead == null ? null : lastRead.next();
- }
-
- /**
- * Obtains PageMemory reference from cache descriptor instead of cache context.
- *
- * @param grpId Cache group id.
- * @return PageMemoryEx instance.
- * @throws IgniteCheckedException if no MemoryPolicy is configured for a name obtained from cache descriptor.
- */
- private PageMemoryEx getPageMemoryForCacheGroup(int grpId) throws IgniteCheckedException {
- // TODO IGNITE-5075: cache descriptor can be removed.
- GridCacheSharedContext sharedCtx = context();
-
- String memPlcName = sharedCtx
- .cache()
- .cacheGroupDescriptors().get(grpId)
- .config()
- .getMemoryPolicyName();
-
- return (PageMemoryEx)sharedCtx.database().memoryPolicy(memPlcName).pageMemory();
- }
-
- /**
- * @param status Last registered checkpoint status.
- * @throws IgniteCheckedException If failed to apply updates.
- * @throws StorageException If IO exception occurred while reading write-ahead log.
- */
- private void applyLastUpdates(CheckpointStatus status) throws IgniteCheckedException {
- if (log.isInfoEnabled())
- log.info("Applying lost cache updates since last checkpoint record [lastMarked="
- + status.startPtr + ", lastCheckpointId=" + status.cpStartId + ']');
-
- cctx.kernalContext().query().skipFieldLookup(true);
-
- long start = U.currentTimeMillis();
- int applied = 0;
-
- try (WALIterator it = cctx.wal().replay(status.startPtr)) {
- Map<T2<Integer, Integer>, T2<Integer, Long>> partStates = new HashMap<>();
-
- while (it.hasNextX()) {
- IgniteBiTuple<WALPointer, WALRecord> next = it.nextX();
-
- WALRecord rec = next.get2();
-
- switch (rec.type()) {
- case DATA_RECORD:
- DataRecord dataRec = (DataRecord)rec;
-
- for (DataEntry dataEntry : dataRec.writeEntries()) {
- int cacheId = dataEntry.cacheId();
-
- GridCacheContext cacheCtx = cctx.cacheContext(cacheId);
-
- applyUpdate(cacheCtx, dataEntry);
-
- applied++;
- }
-
- break;
-
- case PART_META_UPDATE_STATE:
- PartitionMetaStateRecord metaStateRecord = (PartitionMetaStateRecord)rec;
-
- partStates.put(new T2<>(metaStateRecord.cacheId(), metaStateRecord.partitionId()),
- new T2<>((int)metaStateRecord.state(), metaStateRecord.updateCounter()));
-
- break;
-
- default:
- // Skip other records.
- }
- }
-
- restorePartitionState(partStates);
- }
- finally {
- cctx.kernalContext().query().skipFieldLookup(false);
- }
-
- if (log.isInfoEnabled())
- log.info("Finished applying WAL changes [updatesApplied=" + applied +
- ", time=" + (U.currentTimeMillis() - start) + "ms]");
- }
-
- /**
- * @param partStates Partition states.
- * @throws IgniteCheckedException If failed to restore.
- */
- private void restorePartitionState(
- Map<T2<Integer, Integer>, T2<Integer, Long>> partStates
- ) throws IgniteCheckedException {
- for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
- int grpId = grp.groupId();
-
- PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
-
- for (int i = 0; i < grp.affinity().partitions(); i++) {
- if (storeMgr.exists(grpId, i)) {
- storeMgr.ensure(grpId, i);
-
- if (storeMgr.pages(grpId, i) <= 1)
- continue;
-
- long partMetaId = pageMem.partitionMetaPageId(grpId, i);
- long partMetaPage = pageMem.acquirePage(grpId, partMetaId);
- try {
- long pageAddr = pageMem.writeLock(grpId, partMetaId, partMetaPage);
-
- boolean changed = false;
-
- try {
- PagePartitionMetaIO io = PagePartitionMetaIO.VERSIONS.forPage(pageAddr);
-
- T2<Integer, Long> fromWal = partStates.get(new T2<>(grpId, i));
-
- GridDhtLocalPartition part = grp.topology()
- .localPartition(i, AffinityTopologyVersion.NONE, true);
-
- assert part != null;
-
- if (fromWal != null) {
- int stateId = fromWal.get1();
-
- io.setPartitionState(pageAddr, (byte)stateId);
-
- changed = updateState(part, stateId);
-
- if (stateId == GridDhtPartitionState.OWNING.ordinal()) {
- grp.offheap().onPartitionInitialCounterUpdated(i, fromWal.get2());
-
- if (part.initialUpdateCounter() < fromWal.get2()) {
- part.initialUpdateCounter(fromWal.get2());
-
- changed = true;
- }
- }
- }
- else
- changed = updateState(part, (int)io.getPartitionState(pageAddr));
- }
- finally {
- pageMem.writeUnlock(grpId, partMetaId, partMetaPage, null, changed);
- }
- }
- finally {
- pageMem.releasePage(grpId, partMetaId, partMetaPage);
- }
- }
- }
- }
- }
-
- /**
- * @param part Partition to restore state for.
- * @param stateId State enum ordinal.
- * @return Updated flag.
- */
- private boolean updateState(GridDhtLocalPartition part, int stateId) {
- if (stateId != -1) {
- GridDhtPartitionState state = GridDhtPartitionState.fromOrdinal(stateId);
-
- assert state != null;
-
- part.restoreState(state == GridDhtPartitionState.EVICTED ? GridDhtPartitionState.RENTING : state);
-
- return true;
- }
-
- return false;
- }
-
- /**
- * @param cacheCtx Cache context to apply an update.
- * @param dataEntry Data entry to apply.
- */
- private void applyUpdate(GridCacheContext cacheCtx, DataEntry dataEntry) throws IgniteCheckedException {
- GridDhtLocalPartition locPart = cacheCtx.topology()
- .localPartition(dataEntry.partitionId(), AffinityTopologyVersion.NONE, true);
-
- switch (dataEntry.op()) {
- case CREATE:
- case UPDATE:
- cacheCtx.offheap().update(
- cacheCtx,
- dataEntry.key(),
- dataEntry.value(),
- dataEntry.writeVersion(),
- 0L,
- locPart,
- null);
-
- if (dataEntry.partitionCounter() != 0)
- cacheCtx.offheap().onPartitionInitialCounterUpdated(dataEntry.partitionId(), dataEntry.partitionCounter());
-
- break;
-
- case DELETE:
- cacheCtx.offheap().remove(cacheCtx, dataEntry.key(), dataEntry.partitionId(), locPart);
-
- if (dataEntry.partitionCounter() != 0)
- cacheCtx.offheap().onPartitionInitialCounterUpdated(dataEntry.partitionId(), dataEntry.partitionCounter());
-
- break;
-
- default:
- throw new IgniteCheckedException("Invalid operation for WAL entry update: " + dataEntry.op());
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- private void finalizeCheckpointOnRecovery(long cpTs, UUID cpId, WALPointer walPtr) throws IgniteCheckedException {
- assert cpTs != 0;
-
- ByteBuffer tmpWriteBuf = ByteBuffer.allocateDirect(pageSize());
-
- long start = System.currentTimeMillis();
-
- Collection<MemoryPolicy> memPolicies = context().database().memoryPolicies();
-
- List<IgniteBiTuple<PageMemory, Collection<FullPageId>>> cpEntities = new ArrayList<>(memPolicies.size());
-
- for (MemoryPolicy memPlc : memPolicies) {
- PageMemoryEx pageMem = (PageMemoryEx) memPlc.pageMemory();
- cpEntities.add(new IgniteBiTuple<PageMemory, Collection<FullPageId>>(pageMem,
- (pageMem).beginCheckpoint()));
- }
-
- tmpWriteBuf.order(ByteOrder.nativeOrder());
-
- // Identity stores set.
- Collection<PageStore> updStores = new HashSet<>();
-
- int cpPagesCnt = 0;
-
- for (IgniteBiTuple<PageMemory, Collection<FullPageId>> e : cpEntities) {
- PageMemoryEx pageMem = (PageMemoryEx) e.get1();
-
- Collection<FullPageId> cpPages = e.get2();
-
- cpPagesCnt += cpPages.size();
-
- for (FullPageId fullId : cpPages) {
- tmpWriteBuf.rewind();
-
- Integer tag = pageMem.getForCheckpoint(fullId, tmpWriteBuf, null);
-
- if (tag != null) {
- tmpWriteBuf.rewind();
-
- PageStore store = storeMgr.writeInternal(fullId.cacheId(), fullId.pageId(), tmpWriteBuf, tag);
-
- tmpWriteBuf.rewind();
-
- updStores.add(store);
- }
- }
- }
-
- long written = U.currentTimeMillis();
-
- for (PageStore updStore : updStores)
- updStore.sync();
-
- long fsync = U.currentTimeMillis();
-
- for (IgniteBiTuple<PageMemory, Collection<FullPageId>> e : cpEntities)
- ((PageMemoryEx)e.get1()).finishCheckpoint();
-
- writeCheckpointEntry(
- tmpWriteBuf,
- cpTs,
- cpId,
- walPtr,
- null,
- CheckpointEntryType.END);
-
- cctx.pageStore().finishRecover();
-
- if (log.isInfoEnabled())
- log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
- "pagesWrite=%dms, fsync=%dms, total=%dms]",
- cpId,
- cpPagesCnt,
- walPtr,
- written - start,
- fsync - written,
- fsync - start));
- }
-
- /**
- * @param cpId Checkpoint ID.
- * @param ptr Wal pointer of current checkpoint.
- */
- private CheckpointEntry writeCheckpointEntry(
- ByteBuffer tmpWriteBuf,
- long cpTs,
- UUID cpId,
- WALPointer ptr,
- CheckpointRecord rec,
- CheckpointEntryType type
- ) throws IgniteCheckedException {
- assert ptr instanceof FileWALPointer;
-
- FileWALPointer filePtr = (FileWALPointer)ptr;
-
- String fileName = checkpointFileName(cpTs, cpId, type);
-
- try (FileChannel ch = FileChannel.open(Paths.get(cpDir.getAbsolutePath(), fileName),
- StandardOpenOption.CREATE_NEW, StandardOpenOption.APPEND)) {
-
- tmpWriteBuf.rewind();
-
- tmpWriteBuf.putLong(filePtr.index());
-
- tmpWriteBuf.putInt(filePtr.fileOffset());
-
- tmpWriteBuf.putInt(filePtr.length());
-
- tmpWriteBuf.flip();
-
- ch.write(tmpWriteBuf);
-
- tmpWriteBuf.clear();
-
- if (!skipSync)
- ch.force(true);
-
- return type == CheckpointEntryType.START ?
- new CheckpointEntry(cpTs, ptr, cpId, rec.cacheGroupStates()) : null;
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-
- /**
- * @param cpTs Checkpoint timestamp.
- * @param cpId Checkpoint ID.
- * @param type Checkpoint type.
- * @return Checkpoint file name.
- */
- private static String checkpointFileName(long cpTs, UUID cpId, CheckpointEntryType type) {
- return cpTs + "-" + cpId + "-" + type + ".bin";
- }
-
- /**
- *
- */
- @SuppressWarnings("NakedNotify")
- public class Checkpointer extends GridWorker {
- /** Temporary write buffer. */
- private final ByteBuffer tmpWriteBuf;
-
- /** Next scheduled checkpoint progress. */
- private volatile CheckpointProgress scheduledCp;
-
- /** Current checkpoint. This field is updated only by checkpoint thread. */
- private volatile CheckpointProgress curCpProgress;
-
- /** Shutdown now. */
- private volatile boolean shutdownNow;
-
- /**
- * @param gridName Grid name.
- * @param name Thread name.
- * @param log Logger.
- */
- protected Checkpointer(@Nullable String gridName, String name, IgniteLogger log) {
- super(gridName, name, log);
-
- scheduledCp = new CheckpointProgress(U.currentTimeMillis() + checkpointFreq);
-
- tmpWriteBuf = ByteBuffer.allocateDirect(pageSize());
-
- tmpWriteBuf.order(ByteOrder.nativeOrder());
- }
-
- /** {@inheritDoc} */
- @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
- while (!isCancelled()) {
- waitCheckpointEvent();
-
- GridFutureAdapter<Void> enableChangeApplied = GridCacheDatabaseSharedManager.this.enableChangeApplied;
-
- if (enableChangeApplied != null) {
- enableChangeApplied.onDone();
-
- GridCacheDatabaseSharedManager.this.enableChangeApplied = null;
- }
-
- if (checkpointsEnabled)
- doCheckpoint();
- else {
- synchronized (this) {
- scheduledCp.nextCpTs = U.currentTimeMillis() + checkpointFreq;
- }
- }
- }
-
- // Final run after the cancellation.
- if (checkpointsEnabled && !shutdownNow)
- doCheckpoint();
-
- scheduledCp.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
- }
-
- /**
- *
- */
- private CheckpointProgressSnapshot wakeupForCheckpoint(long delayFromNow, String reason) {
- CheckpointProgress sched = scheduledCp;
-
- long next = U.currentTimeMillis() + delayFromNow;
-
- if (sched.nextCpTs <= next)
- return new CheckpointProgressSnapshot(sched);
-
- CheckpointProgressSnapshot ret;
-
- synchronized (this) {
- sched = scheduledCp;
-
- if (sched.nextCpTs > next) {
- sched.reason = reason;
-
- sched.nextCpTs = next;
- }
-
- ret = new CheckpointProgressSnapshot(sched);
-
- notifyAll();
- }
-
- return ret;
- }
-
- /**
- * @param snapshotOperation Snapshot operation.
- */
- public IgniteInternalFuture wakeupForSnapshotCreation(SnapshotOperation snapshotOperation) {
- GridFutureAdapter<Object> ret;
-
- synchronized (this) {
- scheduledCp.nextCpTs = U.currentTimeMillis();
-
- scheduledCp.reason = "snapshot";
-
- scheduledCp.nextSnapshot = true;
-
- scheduledCp.snapshotOperation = snapshotOperation;
-
- ret = scheduledCp.cpBeginFut;
-
- notifyAll();
- }
-
- return ret;
- }
-
- /**
- *
- */
- private void doCheckpoint() {
- try {
- CheckpointMetricsTracker tracker = new CheckpointMetricsTracker();
-
- Checkpoint chp = markCheckpointBegin(tracker);
-
- if (chp.cpPages == null)
- return;
-
- snapshotMgr.onCheckPointBegin();
-
- boolean interrupted = true;
-
- try {
- if (chp.hasDelta()) {
- // Identity stores set.
- GridConcurrentHashSet<PageStore> updStores = new GridConcurrentHashSet<>();
-
- CountDownFuture doneWriteFut = new CountDownFuture(
- asyncRunner == null ? 1 : chp.cpPages.collectionsSize());
-
- tracker.onPagesWriteStart();
-
- if (asyncRunner != null) {
- for (int i = 0; i < chp.cpPages.collectionsSize(); i++) {
- Runnable write = new WriteCheckpointPages(
- tracker,
- chp.cpPages.innerCollection(i),
- updStores,
- doneWriteFut
- );
-
- try {
- asyncRunner.execute(write);
- }
- catch (RejectedExecutionException ignore) {
- // Run the task synchronously.
- write.run();
- }
- }
- }
- else {
- // Single-threaded checkpoint.
- Runnable write = new WriteCheckpointPages(tracker, chp.cpPages, updStores, doneWriteFut);
-
- write.run();
- }
-
- // Wait and check for errors.
- doneWriteFut.get();
-
- // Must re-check shutdown flag here because threads may have skipped some pages.
- // If so, we should not put finish checkpoint mark.
- if (shutdownNow) {
- chp.progress.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
-
- return;
- }
-
- snapshotMgr.afterCheckpointPageWritten();
-
- tracker.onFsyncStart();
-
- if (!skipSync) {
- for (PageStore updStore : updStores) {
- if (shutdownNow) {
- chp.progress.cpFinishFut.onDone(new NodeStoppingException("Node is stopping."));
-
- return;
- }
-
- updStore.sync();
- }
- }
- }
- else {
- tracker.onPagesWriteStart();
- tracker.onFsyncStart();
- }
-
- // Must mark successful checkpoint only if there are no exceptions or interrupts.
- interrupted = false;
- }
- finally {
- if (!interrupted)
- markCheckpointEnd(chp);
- }
-
- tracker.onEnd();
-
- if (chp.hasDelta()) {
- if (printCheckpointStats) {
- if (log.isInfoEnabled())
- log.info(String.format("Checkpoint finished [cpId=%s, pages=%d, markPos=%s, " +
- "walSegmentsCleared=%d, markDuration=%dms, pagesWrite=%dms, fsync=%dms, " +
- "total=%dms]",
- chp.cpEntry.checkpointId(),
- chp.pagesSize,
- chp.cpEntry.checkpointMark(),
- chp.walFilesDeleted,
- tracker.markDuration(),
- tracker.pagesWriteDuration(),
- tracker.fsyncDuration(),
- tracker.totalDuration()));
- }
-
- persStoreMetrics.onCheckpoint(
- tracker.lockWaitDuration(),
- tracker.markDuration(),
- tracker.pagesWriteDuration(),
- tracker.fsyncDuration(),
- tracker.totalDuration(),
- chp.pagesSize,
- tracker.dataPagesWritten(),
- tracker.cowPagesWritten());
- }
- else {
- persStoreMetrics.onCheckpoint(
- tracker.lockWaitDuration(),
- tracker.markDuration(),
- tracker.pagesWriteDuration(),
- tracker.fsyncDuration(),
- tracker.totalDuration(),
- chp.pagesSize,
- tracker.dataPagesWritten(),
- tracker.cowPagesWritten());
- }
- }
- catch (IgniteCheckedException e) {
- // TODO-ignite-db how to handle exception?
- U.error(log, "Failed to create checkpoint.", e);
- }
- }
-
- /**
- *
- */
- @SuppressWarnings("WaitNotInLoop")
- private void waitCheckpointEvent() {
- boolean cancel = false;
-
- try {
- long now = U.currentTimeMillis();
-
- synchronized (this) {
- long remaining;
-
- while ((remaining = scheduledCp.nextCpTs - now) > 0 && !isCancelled()) {
- wait(remaining);
-
- now = U.currentTimeMillis();
- }
- }
- }
- catch (InterruptedException ignored) {
- Thread.currentThread().interrupt();
-
- cancel = true;
- }
-
- if (cancel)
- isCancelled = true;
- }
-
- /**
- *
- */
- @SuppressWarnings("TooBroadScope")
- private Checkpoint markCheckpointBegin(CheckpointMetricsTracker tracker) throws IgniteCheckedException {
- CheckpointRecord cpRec = new CheckpointRecord(null, false);
-
- WALPointer cpPtr = null;
-
- GridMultiCollectionWrapper<FullPageId> cpPages;
-
- final CheckpointProgress curr;
-
- tracker.onLockWaitStart();
-
- checkpointLock.writeLock().lock();
-
- try {
- tracker.onMarkStart();
-
- synchronized (this) {
- curr = scheduledCp;
-
- curr.started = true;
-
- if (curr.reason == null)
- curr.reason = "timeout";
-
- // It is important that we assign a new progress object before checkpoint mark in page memory.
- scheduledCp = new CheckpointProgress(U.currentTimeMillis() + checkpointFreq);
-
- curCpProgress = curr;
- }
-
- final NavigableMap<T2<Integer, Integer>, T2<Integer, Integer>> map =
- new TreeMap<>(FullPageIdIterableComparator.INSTANCE);
-
- DbCheckpointListener.Context ctx0 = new DbCheckpointListener.Context() {
- @Override public boolean nextSnapshot() {
- return curr.nextSnapshot;
- }
-
- @Override public Map<T2<Integer, Integer>, T2<Integer, Integer>> partitionStatMap() {
- return map;
- }
- };
-
- // Listeners must be invoked before we write checkpoint record to WAL.
- for (DbCheckpointListener lsnr : lsnrs)
- lsnr.onCheckpointBegin(ctx0);
-
- for (CacheGroupContext grp : cctx.cache().cacheGroups()) {
- if (grp.isLocal())
- continue;
-
- List<GridDhtLocalPartition> locParts = new ArrayList<>();
-
- for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
- locParts.add(part);
-
- Collections.sort(locParts, ASC_PART_COMPARATOR);
-
- CacheState state = new CacheState(locParts.size());
-
- for (GridDhtLocalPartition part : grp.topology().currentLocalPartitions())
- state.addPartitionState(part.id(), part.dataStore().fullSize(), part.lastAppliedUpdate());
-
- cpRec.addCacheGroupState(grp.groupId(), state);
- }
-
- if (curr.nextSnapshot)
- snapshotMgr.onMarkCheckPointBegin(curr.snapshotOperation, map);
-
- IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> tup = beginAllCheckpoints();
-
- // Todo it maybe more optimally
- Collection<FullPageId> cpPagesList = new ArrayList<>(tup.get2());
-
- for (GridMultiCollectionWrapper<FullPageId> col : tup.get1()) {
- for (int i = 0; i < col.collectionsSize(); i++)
- cpPagesList.addAll(col.innerCollection(i));
- }
-
- cpPages = new GridMultiCollectionWrapper<>(cpPagesList);
-
- if (!F.isEmpty(cpPages)) {
- // No page updates for this checkpoint are allowed from now on.
- cpPtr = cctx.wal().log(cpRec);
-
- if (cpPtr == null)
- cpPtr = CheckpointStatus.NULL_PTR;
- }
- }
- finally {
- checkpointLock.writeLock().unlock();
-
- tracker.onLockRelease();
- }
-
- curr.cpBeginFut.onDone();
-
- if (!F.isEmpty(cpPages)) {
- assert cpPtr != null;
-
- // Sync log outside the checkpoint write lock.
- cctx.wal().fsync(cpPtr);
-
- long cpTs = System.currentTimeMillis();
-
- CheckpointEntry cpEntry = writeCheckpointEntry(
- tmpWriteBuf,
- cpTs,
- cpRec.checkpointId(),
- cpPtr,
- cpRec,
- CheckpointEntryType.START);
-
- checkpointHist.addCheckpointEntry(cpEntry);
-
- if (printCheckpointStats)
- if (log.isInfoEnabled())
- log.info(String.format("Checkpoint started [checkpointId=%s, startPtr=%s, checkpointLockWait=%dms, " +
- "checkpointLockHoldTime=%dms, pages=%d, reason='%s']",
- cpRec.checkpointId(),
- cpPtr,
- tracker.lockWaitDuration(),
- tracker.lockHoldDuration(),
- cpPages.size(),
- curr.reason)
- );
-
- return new Checkpoint(cpEntry, cpPages, curr);
- }
- else {
- if (printCheckpointStats) {
- if (log.isInfoEnabled())
- LT.info(log, String.format("Skipping checkpoint (no pages were modified) [" +
- "checkpointLockWait=%dms, checkpointLockHoldTime=%dms, reason='%s']",
- tracker.lockWaitDuration(),
- tracker.lockHoldDuration(),
- curr.reason));
- }
-
- return new Checkpoint(null, null, curr);
- }
- }
-
- /**
- * @return tuple with collections of FullPageIds obtained from each PageMemory and overall number of dirty pages.
- */
- private IgniteBiTuple<Collection<GridMultiCollectionWrapper<FullPageId>>, Integer> beginAllCheckpoints() {
- Collection<GridMultiCollectionWrapper<FullPageId>> res = new ArrayList(memoryPolicies().size());
-
- int pagesNum = 0;
-
- for (MemoryPolicy memPlc : memoryPolicies()) {
- GridMultiCollectionWrapper<FullPageId> nextCpPagesCol = ((PageMemoryEx) memPlc.pageMemory()).beginCheckpoint();
-
- pagesNum += nextCpPagesCol.size();
-
- res.add(nextCpPagesCol);
- }
-
- return new IgniteBiTuple<>(res, pagesNum);
- }
-
- /**
- * @param chp Checkpoint snapshot.
- */
- private void markCheckpointEnd(Checkpoint chp) throws IgniteCheckedException {
- synchronized (this) {
- for (MemoryPolicy memPlc : memoryPolicies())
- ((PageMemoryEx)memPlc.pageMemory()).finishCheckpoint();
-
- if (chp.hasDelta())
- writeCheckpointEntry(
- tmpWriteBuf,
- chp.cpEntry.checkpointTimestamp(),
- chp.cpEntry.checkpointId(),
- chp.cpEntry.checkpointMark(),
- null,
- CheckpointEntryType.END);
- }
-
- checkpointHist.onCheckpointFinished(chp);
-
- chp.progress.cpFinishFut.onDone();
- }
-
- /** {@inheritDoc} */
- @Override public void cancel() {
- if (log.isDebugEnabled())
- log.debug("Cancelling grid runnable: " + this);
-
- // Do not interrupt runner thread.
- isCancelled = true;
-
- synchronized (this) {
- notifyAll();
- }
- }
-
- /**
- *
- */
- public void shutdownNow() {
- shutdownNow = true;
-
- if (!isCancelled)
- cancel();
- }
- }
-
- /**
- *
- */
- private class WriteCheckpointPages implements Runnable {
- /** */
- private CheckpointMetricsTracker tracker;
-
- /** */
- private Collection<FullPageId> writePageIds;
-
- /** */
- private GridConcurrentHashSet<PageStore> updStores;
-
- /** */
- private CountDownFuture doneFut;
-
- /**
- * @param writePageIds Write page IDs.
- */
- private WriteCheckpointPages(
- CheckpointMetricsTracker tracker,
- Collection<FullPageId> writePageIds,
- GridConcurrentHashSet<PageStore> updStores,
- CountDownFuture doneFut
- ) {
- this.tracker = tracker;
- this.writePageIds = writePageIds;
- this.updStores = updStores;
- this.doneFut = doneFut;
- }
-
- /** {@inheritDoc} */
- @Override public void run() {
- ByteBuffer tmpWriteBuf = threadBuf.get();
-
- long writeAddr = GridUnsafe.bufferAddress(tmpWriteBuf);
-
- snapshotMgr.beforeCheckpointPageWritten();
-
- try {
- for (FullPageId fullId : writePageIds) {
- if (checkpointer.shutdownNow)
- break;
-
- tmpWriteBuf.rewind();
-
- snapshotMgr.beforePageWrite(fullId);
-
- int grpId = fullId.cacheId();
-
- CacheGroupContext grp = context().cache().cacheGroup(grpId);
-
- if (grp == null)
- continue;
-
- PageMemoryEx pageMem = (PageMemoryEx)grp.memoryPolicy().pageMemory();
-
- Integer tag = pageMem.getForCheckpoint(fullId, tmpWriteBuf, persStoreMetrics.metricsEnabled() ? tracker : null);
-
- if (tag != null) {
- tmpWriteBuf.rewind();
-
- if (persStoreMetrics.metricsEnabled()) {
- int pageType = PageIO.getType(tmpWriteBuf);
-
- if (PageIO.isDataPageType(pageType))
- tracker.onDataPageWritten();
- }
-
- if (!skipCrc) {
- PageIO.setCrc(writeAddr, PureJavaCrc32.calcCrc32(tmpWriteBuf, pageSize()));
-
- tmpWriteBuf.rewind();
- }
-
- snapshotMgr.onPageWrite(fullId, tmpWriteBuf);
-
- tmpWriteBuf.rewind();
-
- PageIO.setCrc(writeAddr, 0);
-
- PageStore store = storeMg
<TRUNCATED>
[21/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
new file mode 100644
index 0000000..205e0fc
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRecoveryTxLogicalRecordsTest.java
@@ -0,0 +1,951 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.Ignition;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheRebalanceMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.cache.query.QueryCursor;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
+import org.apache.ignite.cache.query.annotations.QuerySqlField;
+import org.apache.ignite.configuration.BinaryConfiguration;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.pagemem.store.PageStore;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheOffheapManager;
+import org.apache.ignite.internal.processors.cache.IgniteRebalanceIterator;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.FreeListImpl;
+import org.apache.ignite.internal.processors.cache.persistence.freelist.PagesList;
+import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseListImpl;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.junit.Assert;
+
+/**
+ *
+ */
+public class WalRecoveryTxLogicalRecordsTest extends GridCommonAbstractTest {
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache";
+
+ /** Cache 2 name. */
+ private static final String CACHE2_NAME = "cache2";
+
+ /** */
+ public static final int PARTS = 32;
+
+ /** */
+ public static final int WAL_HIST_SIZE = 30;
+
+ /** */
+ private int pageSize = 4 * 1024;
+
+ /** */
+ private CacheConfiguration<?, ?> extraCcfg;
+
+ /** */
+ private Long checkpointFreq;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ CacheConfiguration<Integer, IndexedValue> ccfg = new CacheConfiguration<>(CACHE_NAME);
+
+ ccfg.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg.setRebalanceMode(CacheRebalanceMode.SYNC);
+ ccfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+ ccfg.setIndexedTypes(Integer.class, IndexedValue.class);
+
+ if (extraCcfg != null)
+ cfg.setCacheConfiguration(ccfg, new CacheConfiguration<>(extraCcfg));
+ else
+ cfg.setCacheConfiguration(ccfg);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setPageSize(pageSize);
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ PersistentStoreConfiguration pCfg = new PersistentStoreConfiguration();
+
+ pCfg.setWalHistorySize(WAL_HIST_SIZE);
+
+ if (checkpointFreq != null)
+ pCfg.setCheckpointingFrequency(checkpointFreq);
+
+ cfg.setPersistentStoreConfiguration(pCfg);
+
+ cfg.setMarshaller(null);
+
+ BinaryConfiguration binCfg = new BinaryConfiguration();
+
+ binCfg.setCompactFooter(false);
+
+ cfg.setBinaryConfiguration(binCfg);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalTxSimple() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
+
+ int txCnt = 100;
+
+ int keysPerTx = 10;
+
+ for (int i = 0; i < txCnt; i++) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ cache.put(k, new IndexedValue(k));
+ }
+
+ tx.commit();
+ }
+ }
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ assertEquals(k, cache.get(k).value());
+ }
+ }
+
+ stopGrid();
+
+ ignite = startGrid();
+
+ cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ assertEquals(k, cache.get(k).value());
+ }
+ }
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ QueryCursor<List<?>> cur = cache.query(
+ new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
+
+ List<List<?>> vals = cur.getAll();
+
+ assertEquals(vals.size(), 1);
+ assertEquals("string-" + k, vals.get(0).get(0));
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testWalRecoveryRemoves() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ IgniteCache<Integer, IndexedValue> cache = ignite.cache(CACHE_NAME);
+
+ int txCnt = 100;
+
+ int keysPerTx = 10;
+
+ for (int i = 0; i < txCnt; i++) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ cache.put(k, new IndexedValue(k));
+ }
+
+ tx.commit();
+ }
+ }
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ assertEquals(k, cache.get(k).value());
+ }
+ }
+
+ dbMgr.waitForCheckpoint("test");
+ dbMgr.enableCheckpoints(false).get();
+
+ for (int i = 0; i < txCnt / 2; i++) {
+ try (Transaction tx = ignite.transactions().txStart()) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ cache.remove(k);
+ }
+
+ tx.commit();
+ }
+ }
+
+ stopGrid();
+
+ ignite = startGrid();
+
+ cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < txCnt; i++) {
+ for (int j = 0; j < keysPerTx; j++) {
+ int k = i * keysPerTx + j;
+
+ QueryCursor<List<?>> cur = cache.query(
+ new SqlFieldsQuery("select sVal from IndexedValue where iVal=?").setArgs(k));
+
+ List<List<?>> vals = cur.getAll();
+
+ if (i < txCnt / 2) {
+ assertNull(cache.get(k));
+ assertTrue(F.isEmpty(vals));
+ }
+ else {
+ assertEquals(k, cache.get(k).value());
+
+ assertEquals(1, vals.size());
+ assertEquals("string-" + k, vals.get(0).get(0));
+ }
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRebalanceIterator() throws Exception {
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.waitForCheckpoint("test");
+
+ // This number depends on wal history size.
+ int entries = 25;
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
+ IgniteCache<Integer, Integer> cache2 = ignite.cache(CACHE2_NAME);
+
+ for (int i = 0; i < entries; i++) {
+ // Put to partition 0.
+ cache.put(i * PARTS, i * PARTS);
+
+ // Put to partition 1.
+ cache.put(i * PARTS + 1, i * PARTS + 1);
+
+ // Put to another cache.
+ cache2.put(i, i);
+
+ dbMgr.waitForCheckpoint("test");
+ }
+
+ for (int i = 0; i < entries; i++) {
+ assertEquals((Integer)(i * PARTS), cache.get(i * PARTS));
+ assertEquals((Integer)(i * PARTS + 1), cache.get(i * PARTS + 1));
+ assertEquals((Integer)(i), cache2.get(i));
+ }
+
+ GridCacheContext<Object, Object> cctx = ((IgniteEx)ignite).context().cache().cache(CACHE_NAME).context();
+ IgniteCacheOffheapManager offh = cctx.offheap();
+ AffinityTopologyVersion topVer = cctx.affinity().affinityTopologyVersion();
+
+ for (int i = 0; i < entries; i++) {
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(0, topVer, (long)i)) {
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ assertNotNull(it);
+
+ for (int j = i; j < entries; j++) {
+ assertTrue("i=" + i + ", j=" + j, it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ assertFalse(it.hasNext());
+ }
+
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(1, topVer, (long)i)) {
+ assertNotNull(it);
+
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ for (int j = i; j < entries; j++) {
+ assertTrue(it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS + 1, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS + 1, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ assertFalse(it.hasNext());
+ }
+ }
+
+ stopAllGrids();
+
+ // Check that iterator is valid after restart.
+ ignite = startGrid();
+
+ cctx = ((IgniteEx)ignite).context().cache().cache(CACHE_NAME).context();
+ offh = cctx.offheap();
+ topVer = cctx.affinity().affinityTopologyVersion();
+
+ for (int i = 0; i < entries; i++) {
+ long start = System.currentTimeMillis();
+
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(0, topVer, (long)i)) {
+ long end = System.currentTimeMillis();
+
+ info("Time to get iterator: " + (end - start));
+
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ assertNotNull(it);
+
+ start = System.currentTimeMillis();
+
+ for (int j = i; j < entries; j++) {
+ assertTrue("i=" + i + ", j=" + j, it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ end = System.currentTimeMillis();
+
+ info("Time to iterate: " + (end - start));
+
+ assertFalse(it.hasNext());
+ }
+
+ try (IgniteRebalanceIterator it = offh.rebalanceIterator(1, topVer, (long)i)) {
+ assertNotNull(it);
+
+ assertTrue("Not historical for iteration: " + i, it.historical());
+
+ for (int j = i; j < entries; j++) {
+ assertTrue(it.hasNextX());
+
+ CacheDataRow row = it.next();
+
+ assertEquals(j * PARTS + 1, (int)row.key().value(cctx.cacheObjectContext(), false));
+ assertEquals(j * PARTS + 1, (int)row.value().value(cctx.cacheObjectContext(), false));
+ }
+
+ assertFalse(it.hasNext());
+ }
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testCheckpointHistory() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.waitForCheckpoint("test");
+
+ // This number depends on wal history size.
+ int entries = WAL_HIST_SIZE * 2;
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < entries; i++) {
+ // Put to partition 0.
+ cache.put(i * PARTS, i * PARTS);
+
+ // Put to partition 1.
+ cache.put(i * PARTS + 1, i * PARTS + 1);
+
+ dbMgr.waitForCheckpoint("test");
+ }
+
+ GridCacheDatabaseSharedManager.CheckpointHistory hist = dbMgr.checkpointHistory();
+
+ assertTrue(hist.checkpoints().size() <= WAL_HIST_SIZE);
+
+ File cpDir = dbMgr.checkpointDirectory();
+
+ File[] cpFiles = cpDir.listFiles();
+
+ assertTrue(cpFiles.length <= WAL_HIST_SIZE * 2);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testWalAfterPreloading() throws Exception {
+ Ignite ignite = startGrid();
+
+ try {
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ int entries = 100;
+
+ try (IgniteDataStreamer<Integer, Integer> streamer = ignite.dataStreamer(CACHE_NAME)) {
+ for (int i = 0; i < entries; i++)
+ streamer.addData(i, i);
+ }
+
+ IgniteCache<Integer, Integer> cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < entries; i++)
+ assertEquals(new Integer(i), cache.get(i));
+
+ stopGrid();
+
+ ignite = startGrid();
+
+ cache = ignite.cache(CACHE_NAME);
+
+ for (int i = 0; i < entries; i++)
+ assertEquals(new Integer(i), cache.get(i));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryRandomPutRemove() throws Exception {
+ try {
+ pageSize = 1024;
+
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, PARTS));
+
+ Ignite ignite = startGrid(0);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ dbMgr.enableCheckpoints(false).get();
+
+ IgniteCache<Integer, IndexedValue> cache1 = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> cache2 = ignite.cache(CACHE2_NAME);
+
+ final int KEYS1 = 100;
+
+ for (int i = 0; i < KEYS1; i++)
+ cache1.put(i, new IndexedValue(i));
+
+ for (int i = 0; i < KEYS1; i++) {
+ if (i % 2 == 0)
+ cache1.remove(i);
+ }
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < KEYS1; i++) {
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.remove(i);
+ }
+
+ ignite.close();
+
+ ignite = startGrid(0);
+
+ ignite.cache(CACHE_NAME).put(1, new IndexedValue(0));
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryNoPageLost1() throws Exception {
+ recoveryNoPageLost(false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryNoPageLost2() throws Exception {
+ recoveryNoPageLost(true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRecoveryNoPageLost3() throws Exception {
+ try {
+ pageSize = 1024;
+ checkpointFreq = 100L;
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ List<Integer> pages = null;
+
+ for (int iter = 0; iter < 5; iter++) {
+ log.info("Start node: " + iter);
+
+ Ignite ignite = startGrid(0);
+
+ if (pages != null) {
+ List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
+
+ assertEquals("Iter = " + iter, pages, curPags);
+ }
+
+ final IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
+
+ final int ops = ThreadLocalRandom.current().nextInt(10) + 10;
+
+ GridTestUtils.runMultiThreaded(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < ops; i++) {
+ Integer key = rnd.nextInt(1000);
+
+ cache.put(key, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache.remove(key);
+ }
+
+ return null;
+ }
+ }, 10, "update");
+
+ pages = allocatedPages(ignite, CACHE2_NAME);
+
+ Ignition.stop(ignite.name(), false); //will make checkpoint
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param checkpoint Checkpoint enable flag.
+ * @throws Exception If failed.
+ */
+ private void recoveryNoPageLost(boolean checkpoint) throws Exception {
+ try {
+ pageSize = 1024;
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+ extraCcfg.setAffinity(new RendezvousAffinityFunction(false, 32));
+
+ List<Integer> pages = null;
+
+ AtomicInteger cnt = new AtomicInteger();
+
+ for (int iter = 0; iter < 5; iter++) {
+ log.info("Start node: " + iter);
+
+ Ignite ignite = startGrid(0);
+
+ GridCacheDatabaseSharedManager dbMgr = (GridCacheDatabaseSharedManager)((IgniteEx)ignite).context()
+ .cache().context().database();
+
+ if (!checkpoint)
+ dbMgr.enableCheckpoints(false).get();
+
+ if (pages != null) {
+ List<Integer> curPags = allocatedPages(ignite, CACHE2_NAME);
+
+ assertEquals(pages, curPags);
+ }
+
+ IgniteCache<Integer, Object> cache = ignite.cache(CACHE2_NAME);
+
+ for (int i = 0; i < 128; i++)
+ cache.put(cnt.incrementAndGet(), new byte[256 + iter * 100]);
+
+ pages = allocatedPages(ignite, CACHE2_NAME);
+
+ ignite.close();
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ignite Node.
+ * @param cacheName Cache name.
+ * @return Allocated pages per-store.
+ * @throws Exception If failed.
+ */
+ private List<Integer> allocatedPages(Ignite ignite, String cacheName) throws Exception {
+ FilePageStoreManager storeMgr =
+ (FilePageStoreManager)((IgniteEx)ignite).context().cache().context().pageStore();
+
+ int parts = ignite.affinity(cacheName).partitions();
+
+ List<Integer> res = new ArrayList<>(parts);
+
+ for (int p = 0; p < parts; p++) {
+ PageStore store = storeMgr.getStore(CU.cacheId(cacheName), p);
+
+ store.sync();
+
+ res.add(store.pages());
+ }
+
+ PageStore store = storeMgr.getStore(CU.cacheId(cacheName), PageIdAllocator.INDEX_PARTITION);
+
+ store.sync();
+
+ res.add(store.pages());
+
+ return res;
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testFreeListRecovery() throws Exception {
+ try {
+ pageSize = 1024;
+ extraCcfg = new CacheConfiguration(CACHE2_NAME);
+
+ Ignite ignite = startGrid(0);
+
+ IgniteCache<Integer, IndexedValue> cache1 = ignite.cache(CACHE_NAME);
+ IgniteCache<Object, Object> cache2 = ignite.cache(CACHE2_NAME);
+
+ final int KEYS1 = 2048;
+
+ for (int i = 0; i < KEYS1; i++)
+ cache1.put(i, new IndexedValue(i));
+
+ for (int i = 0; i < KEYS1; i++) {
+ if (i % 2 == 0)
+ cache1.remove(i);
+ }
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int i = 0; i < KEYS1; i++) {
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.put(i, new byte[rnd.nextInt(512)]);
+
+ if (rnd.nextBoolean())
+ cache2.remove(i);
+ }
+
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache1_1 = getFreeListData(ignite, CACHE_NAME);
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache2_1 = getFreeListData(ignite, CACHE2_NAME);
+ T2<long[], Integer> rl1_1 = getReuseListData(ignite, CACHE_NAME);
+ T2<long[], Integer> rl2_1 = getReuseListData(ignite, CACHE2_NAME);
+
+ ignite.close();
+
+ ignite = startGrid(0);
+
+ cache1 = ignite.cache(CACHE_NAME);
+ cache2 = ignite.cache(CACHE2_NAME);
+
+ for (int i = 0; i < KEYS1; i++) {
+ cache1.get(i);
+ cache2.get(i);
+ }
+
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache1_2 = getFreeListData(ignite, CACHE_NAME);
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> cache2_2 = getFreeListData(ignite, CACHE2_NAME);
+ T2<long[], Integer> rl1_2 = getReuseListData(ignite, CACHE_NAME);
+ T2<long[], Integer> rl2_2 = getReuseListData(ignite, CACHE2_NAME);
+
+ checkEquals(cache1_1, cache1_2);
+ checkEquals(cache2_1, cache2_2);
+ checkEquals(rl1_1, rl1_2);
+ checkEquals(rl2_1, rl2_2);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ * @param ignite Node.
+ * @param cacheName Cache name.
+ * @return Cache reuse list data.
+ */
+ private T2<long[], Integer> getReuseListData(Ignite ignite, String cacheName) {
+ GridCacheContext ctx = ((IgniteEx)ignite).context().cache().cache(cacheName).context();
+
+ ReuseListImpl reuseList = GridTestUtils.getFieldValue(ctx.offheap(), "reuseList");
+ PagesList.Stripe[] bucket = GridTestUtils.getFieldValue(reuseList, "bucket");
+
+ long[] ids = null;
+
+ if (bucket != null) {
+ ids = new long[bucket.length];
+
+ for (int i = 0; i < bucket.length; i++)
+ ids[i] = bucket[i].tailId;
+ }
+
+// AtomicIntegerArray cnts = GridTestUtils.getFieldValue(reuseList, PagesList.class, "cnts");
+// assertEquals(1, cnts.length());
+
+ return new T2<>(ids, 0);
+ }
+
+ /**
+ * @param rl1 Data 1 (before stop).
+ * @param rl2 Data 2 (after restore).
+ */
+ private void checkEquals(T2<long[], Integer> rl1, T2<long[], Integer> rl2) {
+ Assert.assertArrayEquals(rl1.get1(), rl2.get1());
+ assertEquals(rl1.get2(), rl2.get2());
+ }
+
+ /**
+ * @param partsLists1 Data 1 (before stop).
+ * @param partsLists2 Data 2 (after restore).
+ */
+ private void checkEquals(Map<Integer, T2<Map<Integer, long[]>, int[]>> partsLists1,
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> partsLists2) {
+ assertEquals(partsLists1.size(), partsLists2.size());
+
+ for (Integer part : partsLists1.keySet()) {
+ T2<Map<Integer, long[]>, int[]> t1 = partsLists1.get(part);
+ T2<Map<Integer, long[]>, int[]> t2 = partsLists2.get(part);
+
+ Map<Integer, long[]> m1 = t1.get1();
+ Map<Integer, long[]> m2 = t2.get1();
+
+ assertEquals(m1.size(), m2.size());
+
+ for (Integer bucket : m1.keySet()) {
+ long tails1[] = m1.get(bucket);
+ long tails2[] = m2.get(bucket);
+
+ Assert.assertArrayEquals(tails1, tails2);
+ }
+
+ Assert.assertArrayEquals("Wrong counts [part=" + part + ']', t1.get2(), t2.get2());
+ }
+ }
+
+ /**
+ * @param ignite Node.
+ * @param cacheName Cache name.
+ * @return Cache free lists data.
+ */
+ private Map<Integer, T2<Map<Integer, long[]>, int[]>> getFreeListData(Ignite ignite, String cacheName) {
+ GridCacheContext ctx = ((IgniteEx)ignite).context().cache().cache(cacheName).context();
+
+ List<GridDhtLocalPartition> parts = ctx.topology().localPartitions();
+
+ assertTrue(!parts.isEmpty());
+ assertEquals(ctx.affinity().partitions(), parts.size());
+
+ Map<Integer, T2<Map<Integer, long[]>, int[]>> res = new HashMap<>();
+
+ boolean foundNonEmpty = false;
+ boolean foundTails = false;
+
+ for (GridDhtLocalPartition part : parts) {
+ FreeListImpl freeList = GridTestUtils.getFieldValue(part.dataStore(), "freeList");
+
+ if (freeList == null)
+ // Lazy store.
+ continue;
+
+ AtomicReferenceArray<PagesList.Stripe[]> buckets = GridTestUtils.getFieldValue(freeList,
+ FreeListImpl.class, "buckets");
+ //AtomicIntegerArray cnts = GridTestUtils.getFieldValue(freeList, PagesList.class, "cnts");
+
+ assertNotNull(buckets);
+ //assertNotNull(cnts);
+ assertTrue(buckets.length() > 0);
+ //assertEquals(cnts.length(), buckets.length());
+
+ Map<Integer, long[]> tailsPerBucket = new HashMap<>();
+
+ for (int i = 0; i < buckets.length(); i++) {
+ PagesList.Stripe[] tails = buckets.get(i);
+
+ long ids[] = null;
+
+ if (tails != null) {
+ ids = new long[tails.length];
+
+ for (int j = 0; j < tails.length; j++)
+ ids[j] = tails[j].tailId;
+ }
+
+ tailsPerBucket.put(i, ids);
+
+ if (tails != null) {
+ assertTrue(tails.length > 0);
+
+ foundTails = true;
+ }
+ }
+
+// int[] cntsPerBucket = new int[cnts.length()];
+//
+// for (int i = 0; i < cnts.length(); i++) {
+// cntsPerBucket[i] = cnts.get(i);
+//
+// if (cntsPerBucket[i] > 0)
+// foundNonEmpty = true;
+// }
+
+ res.put(part.id(), new T2<>(tailsPerBucket, (int[])null));
+ }
+
+ //assertTrue(foundNonEmpty);
+ assertTrue(foundTails);
+
+ return res;
+ }
+
+ /**
+ *
+ */
+ private static class IndexedValue {
+ /** */
+ @QuerySqlField(index = true)
+ private int iVal;
+
+ /** */
+ @QuerySqlField
+ private String sVal;
+
+ /**
+ * @param iVal Indexed value.
+ */
+ private IndexedValue(int iVal) {
+ this.iVal = iVal;
+ sVal = "string-" + iVal;
+ }
+
+ /**
+ * @return Value.
+ */
+ private int value() {
+ return iVal;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
new file mode 100644
index 0000000..303f14e
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal.crc;
+
+import junit.framework.TestCase;
+import java.io.EOFException;
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
+import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
+
+/**
+ *
+ */
+public class IgniteDataIntegrityTests extends TestCase {
+ /** File input. */
+ private FileInput fileInput;
+
+ /** Random access file. */
+ private RandomAccessFile randomAccessFile;
+
+ /** {@inheritDoc} */
+ @Override protected void setUp() throws Exception {
+ super.setUp();
+
+ File file = File.createTempFile("integrity", "dat");
+ file.deleteOnExit();
+
+ randomAccessFile = new RandomAccessFile(file, "rw");
+
+ fileInput = new FileInput(randomAccessFile.getChannel(), ByteBuffer.allocate(1024));
+
+ PureJavaCrc32 pureJavaCrc32 = new PureJavaCrc32();
+
+ ByteBuffer buf = ByteBuffer.allocate(1024);
+ ThreadLocalRandom curr = ThreadLocalRandom.current();
+
+ for (int i = 0; i < 1024; i+=16) {
+ buf.putInt(curr.nextInt());
+ buf.putInt(curr.nextInt());
+ buf.putInt(curr.nextInt());
+ buf.position(i);
+ buf.putInt(PureJavaCrc32.calcCrc32(buf, 12));
+ }
+
+ randomAccessFile.write(buf.array());
+ randomAccessFile.getFD().sync();
+ }
+
+ /**
+ *
+ */
+ public void testSuccessfulPath() throws Exception {
+ checkIntegrity();
+ }
+
+ /**
+ *
+ */
+ public void testIntegrityViolationChecking() throws Exception {
+ toggleOneRandomBit(0, 1024 - 16);
+
+ try {
+ checkIntegrity();
+
+ fail();
+ } catch (IgniteDataIntegrityViolationException ex) {
+ //success
+ }
+ }
+
+ /**
+ *
+ */
+ public void testSkipingLastCorruptedEntry() throws Exception {
+ toggleOneRandomBit(1024 - 16, 1024);
+
+ try {
+ checkIntegrity();
+
+ fail();
+ } catch (EOFException ex) {
+ //success
+ }
+ }
+
+ /**
+ * @param rangeFrom Range from.
+ * @param rangeTo Range to.
+ */
+ private void toggleOneRandomBit(int rangeFrom, int rangeTo) throws IOException {
+ int pos = ThreadLocalRandom.current().nextInt(rangeFrom, rangeTo);
+ randomAccessFile.seek(pos);
+
+ byte b = randomAccessFile.readByte();
+
+ b ^= (1 << 3);
+
+ randomAccessFile.seek(pos);
+ randomAccessFile.writeByte(b);
+ randomAccessFile.getFD().sync();
+ }
+
+ /**
+ *
+ */
+ private void checkIntegrity() throws Exception {
+ randomAccessFile.seek(0);
+
+ for (int i = 0; i < 1024 / 16; i++) {
+ try(FileInput.Crc32CheckingFileInput in = fileInput.startRead(false)) {
+ in.readInt();
+ in.readInt();
+ in.readInt();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
new file mode 100644
index 0000000..6f58782
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.database.BPlusTreeSelfTest;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
+ /** {@inheritDoc} */
+ @Override protected PageMemory createPageMemory() throws Exception {
+ long[] sizes = new long[CPUS + 1];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * MB / CPUS;
+
+ sizes[CPUS] = 10 * MB;
+
+ DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ PageMemory mem = new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new CIX3<Long, FullPageId, PageMemoryEx>(){
+ @Override public void applyx(Long aLong, FullPageId fullPageId, PageMemoryEx ex) {
+ }
+ },
+ new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+
+ mem.start();
+
+ return mem;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long acquiredPages() {
+ return ((PageMemoryImpl)pageMem).acquiredPages();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
new file mode 100644
index 0000000..b263d4f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.database.BPlusTreeReuseSelfTest;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest {
+ /** {@inheritDoc} */
+ @Override protected PageMemory createPageMemory() throws Exception {
+ long[] sizes = new long[CPUS + 1];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * MB / CPUS;
+
+ sizes[CPUS] = 10 * MB;
+
+ DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ PageMemory mem = new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullPageId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration())
+ );
+
+ mem.start();
+
+ return mem;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long acquiredPages() {
+ return ((PageMemoryImpl)pageMem).acquiredPages();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
new file mode 100644
index 0000000..1f29549
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/FullPageIdTableTest.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
+import org.apache.ignite.internal.util.typedef.CI2;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class FullPageIdTableTest extends GridCommonAbstractTest {
+ /** */
+ private static final int CACHE_ID_RANGE = 10;
+
+ /** */
+ private static final int PAGE_ID_RANGE = 1000;
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testRandomOperations() throws Exception {
+ long mem = FullPageIdTable.requiredMemory(CACHE_ID_RANGE * PAGE_ID_RANGE);
+
+ UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
+
+ prov.initialize(new long[] {mem});
+
+ DirectMemoryRegion region = prov.nextRegion();
+
+ try {
+ long seed = U.currentTimeMillis();
+
+ info("Seed: " + seed + "L; //");
+
+ Random rnd = new Random(seed);
+
+ FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
+
+ Map<FullPageId, Long> check = new HashMap<>();
+
+ for (int i = 0; i < 10_000; i++) {
+ int cacheId = rnd.nextInt(CACHE_ID_RANGE) + 1;
+ int pageId = rnd.nextInt(PAGE_ID_RANGE);
+
+ FullPageId fullId = new FullPageId(pageId, cacheId);
+
+ boolean put = rnd.nextInt(3) != -1;
+
+ if (put) {
+ long val = rnd.nextLong();
+
+ tbl.put(cacheId, pageId, val, 0);
+ check.put(fullId, val);
+ }
+ else {
+ tbl.remove(cacheId, pageId, 0);
+ check.remove(fullId);
+ }
+
+ verifyLinear(tbl, check);
+
+ if (i > 0 && i % 1000 == 0)
+ info("Done: " + i);
+ }
+ }
+ finally {
+ prov.shutdown();
+ }
+ }
+
+ /**
+ * @param tbl Table to check.
+ * @param check Expected mapping.
+ */
+ private void verifyLinear(FullPageIdTable tbl, Map<FullPageId, Long> check) {
+ final Map<FullPageId, Long> collector = new HashMap<>();
+
+ tbl.visitAll(new CI2<FullPageId, Long>() {
+ @Override public void apply(FullPageId fullId, Long val) {
+ if (collector.put(fullId, val) != null)
+ throw new AssertionError("Duplicate full page ID mapping: " + fullId);
+ }
+ });
+
+ assertEquals("Size check failed", check.size(), collector.size());
+
+ for (Map.Entry<FullPageId, Long> entry : check.entrySet())
+ assertEquals("Mapping comparison failed for key: " + entry.getKey(),
+ entry.getValue(), collector.get(entry.getKey()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
new file mode 100644
index 0000000..d9257bd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.processors.database.MetadataStorageSelfTest;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class MetadataStoragePageMemoryImplTest extends MetadataStorageSelfTest{
+ /** Make sure page is small enough to trigger multiple pages in a linked list. */
+ public static final int PAGE_SIZE = 1024;
+
+ /** */
+ private static File allocationPath;
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ allocationPath = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+ }
+
+ /**
+ * @param clean Clean flag. If {@code true}, will clean previous memory state and allocate
+ * new empty page memory.
+ * @return Page memory instance.
+ */
+ @Override protected PageMemory memory(boolean clean) throws Exception {
+ long[] sizes = new long[10];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * 1024;
+
+ DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), allocationPath);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ return new PageMemoryImpl(
+ provider, sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
new file mode 100644
index 0000000..0ef593f
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridKernalContext;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.StorageException;
+import org.apache.ignite.internal.pagemem.wal.WALIterator;
+import org.apache.ignite.internal.pagemem.wal.WALPointer;
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.lang.IgniteFuture;
+
+/**
+ *
+ */
+public class NoOpWALManager implements IgniteWriteAheadLogManager {
+ /** {@inheritDoc} */
+ @Override public boolean isAlwaysWriteFullPages() {
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean isFullSync() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void resumeLogging(WALPointer ptr) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALPointer log(WALRecord entry) throws IgniteCheckedException, StorageException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void fsync(WALPointer ptr) throws IgniteCheckedException, StorageException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public WALIterator replay(WALPointer start) throws IgniteCheckedException, StorageException {
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void release(WALPointer start) throws IgniteCheckedException {
+ // No-op.
+ }
+
+ /** {@inheritDoc} */
+ @Override public int truncate(WALPointer ptr) {
+ return 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean reserved(WALPointer ptr) {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void start(GridCacheSharedContext cctx) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void stop(boolean cancel) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStart(boolean reconnect) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onKernalStop(boolean cancel) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDisconnected(IgniteFuture reconnectFut) {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void printMemoryStats() {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onActivate(GridKernalContext kctx) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onDeActivate(GridKernalContext kctx) throws IgniteCheckedException {
+
+ }
+
+ /** {@inheritDoc} */
+ @Override public int walArchiveSegments() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
new file mode 100644
index 0000000..1a8aaa4
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageIdDistributionTest.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import org.apache.ignite.internal.mem.DirectMemoryRegion;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdUtils;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.FullPageIdTable;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl;
+import org.apache.ignite.internal.util.typedef.T2;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.logger.java.JavaLogger;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class PageIdDistributionTest extends GridCommonAbstractTest {
+ /** */
+ private static final int[] CACHE_IDS = new int[] {
+ CU.cacheId("partitioned1"),
+ CU.cacheId("partitioned2"),
+ CU.cacheId("partitioned3"),
+ CU.cacheId("partitioned4"),
+ CU.cacheId("replicated1"),
+ CU.cacheId("replicated2"),
+ CU.cacheId("replicated3"),
+ CU.cacheId("replicated4"),
+ };
+
+ /** */
+ private static final int PARTS = 1024;
+
+ /** */
+ private static final int PAGES = 10240;
+
+ /**
+ *
+ */
+ public void testDistributions() {
+ printPageIdDistribution(
+ CU.cacheId("partitioned"), 1024, 30_000, 32, 2.5f);
+
+ printPageIdDistribution(
+ CU.cacheId("partitioned"), 1024, 30_000, 64, 2.5f);
+
+ printPageIdDistribution(
+ CU.cacheId(null), 1024, 30_000, 32, 2.5f);
+ }
+
+ /**
+ * @param cacheId Cache id.
+ * @param parts Parts.
+ * @param pagesPerPartition Pages per partition.
+ * @param segments Segments.
+ * @param capFactor Capacity factor.
+ */
+ private void printPageIdDistribution(
+ int cacheId,
+ int parts,
+ int pagesPerPartition,
+ int segments,
+ float capFactor
+ ) {
+ int allIds = parts * pagesPerPartition;
+
+ int perSegmentSize = allIds / segments;
+ int capacity = (int)(perSegmentSize * capFactor);
+
+ info("Total ids: " + allIds);
+
+ List<Map<Integer, Integer>> collisionsPerSegment = new ArrayList<>(segments);
+
+ for (int i = 0; i < segments; i++)
+ collisionsPerSegment.add(new HashMap<Integer, Integer>(allIds / segments, 1.0f));
+
+ int[] numInSegment = new int[segments];
+
+ for (int p = 0; p < parts; p++) {
+ for (int i = 0; i < pagesPerPartition; i++) {
+ long pageId = PageIdUtils.pageId(p, (byte)0, i);
+
+ int segment = PageMemoryImpl.segmentIndex(cacheId, pageId, segments);
+
+ int idxInSegment = U.safeAbs(FullPageId.hashCode(cacheId, pageId)) % capacity;
+
+ Map<Integer, Integer> idxCollisions = collisionsPerSegment.get(segment);
+
+ Integer old = idxCollisions.get(idxInSegment);
+ idxCollisions.put(idxInSegment, old == null ? 1 : old + 1);
+
+ numInSegment[segment]++;
+ }
+ }
+
+ for (int i = 0; i < collisionsPerSegment.size(); i++) {
+ Map<Integer, Integer> idxCollisions = collisionsPerSegment.get(i);
+
+ int distinctPositions = idxCollisions.size();
+
+ int totalCnt = 0;
+ int nonZero = 0;
+
+ for (Map.Entry<Integer, Integer> collision : idxCollisions.entrySet()) {
+ if (collision.getValue() != null) {
+ totalCnt += collision.getValue();
+ nonZero++;
+ }
+ }
+
+ info(String.format("Segment stats [i=%d, total=%d, distinct=%d, spaceUsed=%d%%, avgItCnt=%.1f + ']",
+ i, numInSegment[i], distinctPositions, distinctPositions * 100 / numInSegment[i],
+ (float)totalCnt / nonZero));
+ }
+
+ info("==========================================================");
+ }
+
+ /**
+ * Uncomment and run this test manually to get data to plot histogram for per-element distance from ideal.
+ * You can use Octave to plot the histogram:
+ * <pre>
+ * all = csvread("histo.txt");
+ * hist(all, 200)
+ * </pre>
+ *
+ * @throws Exception If failed.
+ */
+ public void _testRealHistory() throws Exception {
+ int capacity = CACHE_IDS.length * PARTS * PAGES;
+
+ info("Capacity: " + capacity);
+
+ long mem = FullPageIdTable.requiredMemory(capacity);
+
+ info(U.readableSize(mem, true));
+
+ UnsafeMemoryProvider prov = new UnsafeMemoryProvider(new JavaLogger());
+
+ prov.initialize(new long[] {mem});
+
+ DirectMemoryRegion region = prov.nextRegion();
+
+ try {
+ long seed = U.currentTimeMillis();
+
+ info("Seed: " + seed + "L; //");
+
+ Random rnd = new Random(seed);
+
+ FullPageIdTable tbl = new FullPageIdTable(region.address(), region.size(), true);
+
+ Map<T2<Integer, Integer>, Integer> allocated = new HashMap<>();
+
+ for (int i = 0; i < capacity; i++) {
+ int cacheId = CACHE_IDS[rnd.nextInt(CACHE_IDS.length)];
+ int partId = rnd.nextInt(PARTS);
+
+ T2<Integer, Integer> key = new T2<>(cacheId, partId);
+
+ Integer pageIdx = allocated.get(key);
+
+ pageIdx = pageIdx == null ? 1 : pageIdx + 1;
+
+ if (pageIdx > PAGES)
+ continue;
+
+ tbl.put(cacheId, PageIdUtils.pageId(partId, (byte)0, pageIdx), 1, 0);
+
+ allocated.put(key, pageIdx);
+
+ if (i > 0 && i % 100_000 == 0)
+ info("Done: " + i);
+ }
+
+ int[] scans = new int[capacity];
+
+ int cur = 0;
+
+ for (T2<Integer, Integer> key : allocated.keySet()) {
+ Integer alloc = allocated.get(key);
+
+ if (alloc != null) {
+ for (int idx = 1; idx <= alloc; idx++) {
+ scans[cur] = tbl.distanceFromIdeal(key.get1(), PageIdUtils.pageId(key.get2(), (byte)0, idx), 0);
+
+ assert scans[cur] != -1;
+
+ cur++;
+ }
+ }
+ }
+
+ try (FileOutputStream out = new FileOutputStream("histo.txt")) {
+ PrintWriter w = new PrintWriter(new OutputStreamWriter(out));
+
+ for (int scan : scans) {
+ if (scan != 0)
+ w.println(scan);
+ }
+
+ w.flush();
+ }
+ }
+ finally {
+ prov.shutdown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
new file mode 100644
index 0000000..1fff1f0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.file.MappedFileMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageMemory;
+import org.apache.ignite.internal.pagemem.impl.PageMemoryNoLoadSelfTest;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+
+/**
+ *
+ */
+public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
+ /**
+ * @return Page memory implementation.
+ */
+ @Override protected PageMemory memory() throws Exception {
+ File memDir = U.resolveWorkDirectory(U.defaultWorkDirectory(), "pagemem", false);
+
+ long[] sizes = new long[10];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 5 * 1024 * 1024;
+
+ DirectMemoryProvider provider = new MappedFileMemoryProvider(log(), memDir);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(log),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ return new PageMemoryImpl(
+ provider,
+ sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuffer, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ },
+ new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ }
+
+ /** {@inheritDoc} */
+ @Override public void testPageHandleDeallocation() throws Exception {
+ // No-op.
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
new file mode 100644
index 0000000..0366eca
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.nio.ByteBuffer;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.internal.mem.DirectMemoryProvider;
+import org.apache.ignite.internal.mem.IgniteOutOfMemoryException;
+import org.apache.ignite.internal.mem.unsafe.UnsafeMemoryProvider;
+import org.apache.ignite.internal.pagemem.FullPageId;
+import org.apache.ignite.internal.pagemem.PageIdAllocator;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
+import org.apache.ignite.internal.util.lang.GridInClosure3X;
+import org.apache.ignite.internal.util.typedef.CIX3;
+import org.apache.ignite.testframework.junits.GridTestKernalContext;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.testframework.junits.logger.GridTestLog4jLogger;
+
+/**
+ *
+ */
+public class PageMemoryImplTest extends GridCommonAbstractTest {
+ /** Mb. */
+ private static final long MB = 1024 * 1024;
+
+ /** Page size. */
+ private static final int PAGE_SIZE = 1024;
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testThatAllocationTooMuchPagesCauseToOOMException() throws Exception {
+ PageMemoryImpl memory = createPageMemory();
+
+ try {
+ while (!Thread.currentThread().isInterrupted())
+ memory.allocatePage(1, PageIdAllocator.INDEX_PARTITION, PageIdAllocator.FLAG_IDX);
+ }
+ catch (IgniteOutOfMemoryException ignore) {
+ //Success
+ }
+
+ assertFalse(memory.safeToUpdate());
+ }
+
+ /**
+ *
+ */
+ private PageMemoryImpl createPageMemory() throws Exception {
+ long[] sizes = new long[5];
+
+ for (int i = 0; i < sizes.length; i++)
+ sizes[i] = 1024 * MB / 4;
+
+ sizes[4] = 10 * MB;
+
+ DirectMemoryProvider provider = new UnsafeMemoryProvider(log);
+
+ GridCacheSharedContext<Object, Object> sharedCtx = new GridCacheSharedContext<>(
+ new GridTestKernalContext(new GridTestLog4jLogger()),
+ null,
+ null,
+ null,
+ new NoOpPageStoreManager(),
+ new NoOpWALManager(),
+ new IgniteCacheDatabaseSharedManager(),
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null
+ );
+
+ PageMemoryImpl mem = new PageMemoryImpl(
+ provider,
+ sizes,
+ sharedCtx,
+ PAGE_SIZE,
+ new CIX3<FullPageId, ByteBuffer, Integer>() {
+ @Override public void applyx(FullPageId fullPageId, ByteBuffer byteBuf, Integer tag) {
+ assert false : "No evictions should happen during the test";
+ }
+ },
+ new GridInClosure3X<Long, FullPageId, PageMemoryEx>() {
+ @Override public void applyx(Long page, FullPageId fullId, PageMemoryEx pageMem) {
+ }
+ }, new CheckpointLockStateChecker() {
+ @Override public boolean checkpointLockIsHeldByThread() {
+ return true;
+ }
+ },
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+
+ mem.start();
+
+ return mem;
+ }
+}
[44/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java
deleted file mode 100644
index fadd685..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeListImpl.java
+++ /dev/null
@@ -1,594 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.freelist;
-
-import java.util.concurrent.atomic.AtomicReferenceArray;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.pagemem.PageIdAllocator;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertFragmentRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageInsertRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageRemoveRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageUpdateRecord;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-import org.apache.ignite.internal.processors.cache.database.MemoryMetricsImpl;
-import org.apache.ignite.internal.processors.cache.database.MemoryPolicy;
-import org.apache.ignite.internal.processors.cache.database.evict.PageEvictionTracker;
-import org.apache.ignite.internal.processors.cache.database.tree.io.CacheVersionIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPagePayload;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- */
-public class FreeListImpl extends PagesList implements FreeList, ReuseList {
- /** */
- private static final int BUCKETS = 256; // Must be power of 2.
-
- /** */
- private static final int REUSE_BUCKET = BUCKETS - 1;
-
- /** */
- private static final Integer COMPLETE = Integer.MAX_VALUE;
-
- /** */
- private static final Integer FAIL_I = Integer.MIN_VALUE;
-
- /** */
- private static final Long FAIL_L = Long.MAX_VALUE;
-
- /** */
- private static final int MIN_PAGE_FREE_SPACE = 8;
-
- /** */
- private final int shift;
-
- /** */
- private final AtomicReferenceArray<Stripe[]> buckets = new AtomicReferenceArray<>(BUCKETS);
-
- /** */
- private final int MIN_SIZE_FOR_DATA_PAGE;
-
- /** */
- private final int emptyDataPagesBucket;
-
- /** */
- private final PageHandler<CacheDataRow, Boolean> updateRow = new UpdateRowHandler();
-
- /** */
- private final MemoryMetricsImpl memMetrics;
-
- /** */
- private final PageEvictionTracker evictionTracker;
-
- /**
- *
- */
- private final class UpdateRowHandler extends PageHandler<CacheDataRow, Boolean> {
- @Override public Boolean run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- CacheDataRow row,
- int itemId)
- throws IgniteCheckedException {
- DataPageIO io = (DataPageIO)iox;
-
- int rowSize = getRowSize(row);
-
- boolean updated = io.updateRow(pageAddr, itemId, pageSize(), null, row, rowSize);
-
- evictionTracker.touchPage(pageId);
-
- if (updated && needWalDeltaRecord(pageId, page, walPlc)) {
- // TODO This record must contain only a reference to a logical WAL record with the actual data.
- byte[] payload = new byte[rowSize];
-
- DataPagePayload data = io.readPayload(pageAddr, itemId, pageSize());
-
- assert data.payloadSize() == rowSize;
-
- PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
-
- wal.log(new DataPageUpdateRecord(
- cacheId,
- pageId,
- itemId,
- payload));
- }
-
- return updated;
- }
- }
-
- /** */
- private final PageHandler<CacheDataRow, Integer> writeRow = new WriteRowHandler();
-
- /**
- *
- */
- private final class WriteRowHandler extends PageHandler<CacheDataRow, Integer> {
- @Override public Integer run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- CacheDataRow row,
- int written)
- throws IgniteCheckedException {
- DataPageIO io = (DataPageIO)iox;
-
- int rowSize = getRowSize(row);
- int oldFreeSpace = io.getFreeSpace(pageAddr);
-
- assert oldFreeSpace > 0 : oldFreeSpace;
-
- // If the full row does not fit into this page write only a fragment.
- written = (written == 0 && oldFreeSpace >= rowSize) ? addRow(pageId, page, pageAddr, io, row, rowSize):
- addRowFragment(pageId, page, pageAddr, io, row, written, rowSize);
-
- // Reread free space after update.
- int newFreeSpace = io.getFreeSpace(pageAddr);
-
- if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
- int bucket = bucket(newFreeSpace, false);
-
- put(null, pageId, page, pageAddr, bucket);
- }
-
- if (written == rowSize)
- evictionTracker.touchPage(pageId);
-
- // Avoid boxing with garbage generation for usual case.
- return written == rowSize ? COMPLETE : written;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param row Row.
- * @param rowSize Row size.
- * @return Written size which is always equal to row size here.
- * @throws IgniteCheckedException If failed.
- */
- private int addRow(
- long pageId,
- long page,
- long pageAddr,
- DataPageIO io,
- CacheDataRow row,
- int rowSize
- ) throws IgniteCheckedException {
- io.addRow(pageAddr, row, rowSize, pageSize());
-
- if (needWalDeltaRecord(pageId, page, null)) {
- // TODO This record must contain only a reference to a logical WAL record with the actual data.
- byte[] payload = new byte[rowSize];
-
- DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
-
- assert data.payloadSize() == rowSize;
-
- PageUtils.getBytes(pageAddr, data.offset(), payload, 0, rowSize);
-
- wal.log(new DataPageInsertRecord(
- cacheId,
- pageId,
- payload));
- }
-
- return rowSize;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param row Row.
- * @param written Written size.
- * @param rowSize Row size.
- * @return Updated written size.
- * @throws IgniteCheckedException If failed.
- */
- private int addRowFragment(
- long pageId,
- long page,
- long pageAddr,
- DataPageIO io,
- CacheDataRow row,
- int written,
- int rowSize
- ) throws IgniteCheckedException {
- // Read last link before the fragment write, because it will be updated there.
- long lastLink = row.link();
-
- int payloadSize = io.addRowFragment(pageMem, pageAddr, row, written, rowSize, pageSize());
-
- assert payloadSize > 0 : payloadSize;
-
- if (needWalDeltaRecord(pageId, page, null)) {
- // TODO This record must contain only a reference to a logical WAL record with the actual data.
- byte[] payload = new byte[payloadSize];
-
- DataPagePayload data = io.readPayload(pageAddr, PageIdUtils.itemId(row.link()), pageSize());
-
- PageUtils.getBytes(pageAddr, data.offset(), payload, 0, payloadSize);
-
- wal.log(new DataPageInsertFragmentRecord(cacheId, pageId, payload, lastLink));
- }
-
- return written + payloadSize;
- }
- }
-
-
- /** */
- private final PageHandler<Void, Long> rmvRow = new RemoveRowHandler();
-
- /**
- *
- */
- private final class RemoveRowHandler extends PageHandler<Void, Long> {
- @Override public Long run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- Void ignored,
- int itemId)
- throws IgniteCheckedException {
- DataPageIO io = (DataPageIO)iox;
-
- int oldFreeSpace = io.getFreeSpace(pageAddr);
-
- assert oldFreeSpace >= 0: oldFreeSpace;
-
- long nextLink = io.removeRow(pageAddr, itemId, pageSize());
-
- if (needWalDeltaRecord(pageId, page, walPlc))
- wal.log(new DataPageRemoveRecord(cacheId, pageId, itemId));
-
- int newFreeSpace = io.getFreeSpace(pageAddr);
-
- if (newFreeSpace > MIN_PAGE_FREE_SPACE) {
- int newBucket = bucket(newFreeSpace, false);
-
- if (oldFreeSpace > MIN_PAGE_FREE_SPACE) {
- int oldBucket = bucket(oldFreeSpace, false);
-
- if (oldBucket != newBucket) {
- // It is possible that page was concurrently taken for put, in this case put will handle bucket change.
- if (removeDataPage(pageId, page, pageAddr, io, oldBucket))
- put(null, pageId, page, pageAddr, newBucket);
- }
- }
- else
- put(null, pageId, page, pageAddr, newBucket);
-
- if (io.isEmpty(pageAddr))
- evictionTracker.forgetPage(pageId);
- }
-
- // For common case boxed 0L will be cached inside of Long, so no garbage will be produced.
- return nextLink;
- }
- }
-
- /**
- * @param cacheId Cache ID.
- * @param name Name (for debug purpose).
- * @param memMetrics Memory metrics.
- * @param memPlc Memory policy.
- * @param reuseList Reuse list or {@code null} if this free list will be a reuse list for itself.
- * @param wal Write ahead log manager.
- * @param metaPageId Metadata page ID.
- * @param initNew {@code True} if new metadata should be initialized.
- * @throws IgniteCheckedException If failed.
- */
- public FreeListImpl(
- int cacheId,
- String name,
- MemoryMetricsImpl memMetrics,
- MemoryPolicy memPlc,
- ReuseList reuseList,
- IgniteWriteAheadLogManager wal,
- long metaPageId,
- boolean initNew) throws IgniteCheckedException {
- super(cacheId, name, memPlc.pageMemory(), BUCKETS, wal, metaPageId);
- this.evictionTracker = memPlc.evictionTracker();
- this.reuseList = reuseList == null ? this : reuseList;
- int pageSize = pageMem.pageSize();
-
- assert U.isPow2(pageSize) : "Page size must be a power of 2: " + pageSize;
- assert U.isPow2(BUCKETS);
- assert BUCKETS <= pageSize : pageSize;
-
- // TODO this constant is used because currently we cannot reuse data pages as index pages
- // TODO and vice-versa. It should be removed when data storage format is finalized.
- MIN_SIZE_FOR_DATA_PAGE = pageSize - DataPageIO.MIN_DATA_PAGE_OVERHEAD;
-
- int shift = 0;
-
- while (pageSize > BUCKETS) {
- shift++;
- pageSize >>>= 1;
- }
-
- this.shift = shift;
-
- this.memMetrics = memMetrics;
-
- emptyDataPagesBucket = bucket(MIN_SIZE_FOR_DATA_PAGE, false);
-
- init(metaPageId, initNew);
- }
-
- /**
- * Calculates average fill factor over FreeListImpl instance.
- */
- public float fillFactor() {
- long pageSize = pageSize();
-
- long totalSize = 0;
- long loadSize = 0;
-
- for (int b = BUCKETS - 2; b > 0; b--) {
- long bsize = pageSize - ((REUSE_BUCKET - b) << shift);
-
- long pages = bucketsSize[b].longValue();
-
- loadSize += pages * (pageSize - bsize);
-
- totalSize += pages * pageSize;
- }
-
- return totalSize == 0 ? -1L : ((float) loadSize / totalSize);
- }
-
- /** {@inheritDoc} */
- @Override public void dumpStatistics(IgniteLogger log) {
- long dataPages = 0;
-
- final boolean dumpBucketsInfo = false;
-
- for (int b = 0; b < BUCKETS; b++) {
- long size = bucketsSize[b].longValue();
-
- if (!isReuseBucket(b))
- dataPages += size;
-
- if (dumpBucketsInfo) {
- Stripe[] stripes = getBucket(b);
-
- boolean empty = true;
-
- if (stripes != null) {
- for (Stripe stripe : stripes) {
- if (!stripe.empty) {
- empty = false;
-
- break;
- }
- }
- }
-
- log.info("Bucket [b=" + b +
- ", size=" + size +
- ", stripes=" + (stripes != null ? stripes.length : 0) +
- ", stripesEmpty=" + empty + ']');
- }
- }
-
- if (dataPages > 0) {
- log.info("FreeList [name=" + name +
- ", buckets=" + BUCKETS +
- ", dataPages=" + dataPages +
- ", reusePages=" + bucketsSize[REUSE_BUCKET].longValue() + "]");
- }
- }
-
- /**
- * @param freeSpace Page free space.
- * @param allowReuse {@code True} if it is allowed to get reuse bucket.
- * @return Bucket.
- */
- private int bucket(int freeSpace, boolean allowReuse) {
- assert freeSpace > 0 : freeSpace;
-
- int bucket = freeSpace >>> shift;
-
- assert bucket >= 0 && bucket < BUCKETS : bucket;
-
- if (!allowReuse && isReuseBucket(bucket))
- bucket--;
-
- return bucket;
- }
-
- /**
- * @param part Partition.
- * @return Page ID.
- * @throws IgniteCheckedException If failed.
- */
- private long allocateDataPage(int part) throws IgniteCheckedException {
- assert part <= PageIdAllocator.MAX_PARTITION_ID;
- assert part != PageIdAllocator.INDEX_PARTITION;
-
- return pageMem.allocatePage(cacheId, part, PageIdAllocator.FLAG_DATA);
- }
-
- /** {@inheritDoc} */
- @Override public void insertDataRow(CacheDataRow row) throws IgniteCheckedException {
- int rowSize = getRowSize(row);
-
- int written = 0;
-
- do {
- if (written != 0)
- memMetrics.incrementLargeEntriesPages();
-
- int freeSpace = Math.min(MIN_SIZE_FOR_DATA_PAGE, rowSize - written);
-
- long pageId = 0L;
-
- if (freeSpace == MIN_SIZE_FOR_DATA_PAGE)
- pageId = takeEmptyPage(emptyDataPagesBucket, DataPageIO.VERSIONS);
-
- boolean reuseBucket = false;
-
- // TODO: properly handle reuse bucket.
- if (pageId == 0L) {
- for (int b = bucket(freeSpace, false) + 1; b < BUCKETS - 1; b++) {
- pageId = takeEmptyPage(b, DataPageIO.VERSIONS);
-
- if (pageId != 0L) {
- reuseBucket = isReuseBucket(b);
-
- break;
- }
- }
- }
-
- boolean allocated = pageId == 0L;
-
- if (allocated)
- pageId = allocateDataPage(row.partition());
-
- DataPageIO init = reuseBucket || allocated ? DataPageIO.VERSIONS.latest() : null;
-
- written = write(pageId, writeRow, init, row, written, FAIL_I);
-
- assert written != FAIL_I; // We can't fail here.
- }
- while (written != COMPLETE);
- }
-
- /** {@inheritDoc} */
- @Override public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException {
- assert link != 0;
-
- long pageId = PageIdUtils.pageId(link);
- int itemId = PageIdUtils.itemId(link);
-
- Boolean updated = write(pageId, updateRow, row, itemId, null);
-
- assert updated != null; // Can't fail here.
-
- return updated;
- }
-
- /** {@inheritDoc} */
- @Override public void removeDataRowByLink(long link) throws IgniteCheckedException {
- assert link != 0;
-
- long pageId = PageIdUtils.pageId(link);
- int itemId = PageIdUtils.itemId(link);
-
- long nextLink = write(pageId, rmvRow, itemId, FAIL_L);
-
- assert nextLink != FAIL_L; // Can't fail here.
-
- while (nextLink != 0L) {
- memMetrics.decrementLargeEntriesPages();
-
- itemId = PageIdUtils.itemId(nextLink);
- pageId = PageIdUtils.pageId(nextLink);
-
- nextLink = write(pageId, rmvRow, itemId, FAIL_L);
-
- assert nextLink != FAIL_L; // Can't fail here.
- }
- }
-
- /** {@inheritDoc} */
- @Override protected Stripe[] getBucket(int bucket) {
- return buckets.get(bucket);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd) {
- return buckets.compareAndSet(bucket, exp, upd);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean isReuseBucket(int bucket) {
- return bucket == REUSE_BUCKET;
- }
-
- /**
- * @return Number of empty data pages in free list.
- */
- public int emptyDataPages() {
- return bucketsSize[emptyDataPagesBucket].intValue();
- }
-
- /** {@inheritDoc} */
- @Override public void addForRecycle(ReuseBag bag) throws IgniteCheckedException {
- assert reuseList == this: "not allowed to be a reuse list";
-
- put(bag, 0, 0, 0L, REUSE_BUCKET);
- }
-
- /** {@inheritDoc} */
- @Override public long takeRecycledPage() throws IgniteCheckedException {
- assert reuseList == this: "not allowed to be a reuse list";
-
- return takeEmptyPage(REUSE_BUCKET, null);
- }
-
- /** {@inheritDoc} */
- @Override public long recycledPagesCount() throws IgniteCheckedException {
- assert reuseList == this: "not allowed to be a reuse list";
-
- return storedPagesCount(REUSE_BUCKET);
- }
-
- /**
- * @param row Row.
- * @return Entry size on page.
- * @throws IgniteCheckedException If failed.
- */
- private static int getRowSize(CacheDataRow row) throws IgniteCheckedException {
- int keyLen = row.key().valueBytesLength(null);
- int valLen = row.value().valueBytesLength(null);
-
- return keyLen + valLen + CacheVersionIO.size(row.version(), false) + 8 + (row.cacheId() == 0 ? 0 : 4);
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "FreeList [name=" + name + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
deleted file mode 100644
index a8f3037..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/PagesList.java
+++ /dev/null
@@ -1,1484 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.freelist;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteSystemProperties;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.pagemem.wal.record.delta.DataPageSetFreeListPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PageListMetaResetCountRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListAddPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListInitNewPageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListRemovePageRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetNextRecord;
-import org.apache.ignite.internal.pagemem.wal.record.delta.PagesListSetPreviousRecord;
-import org.apache.ignite.internal.processors.cache.database.DataStructure;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListMetaIO;
-import org.apache.ignite.internal.processors.cache.database.freelist.io.PagesListNodeIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseBag;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.util.GridArrays;
-import org.apache.ignite.internal.util.GridLongList;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.Nullable;
-
-import static java.lang.Boolean.FALSE;
-import static java.lang.Boolean.TRUE;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_DATA;
-import static org.apache.ignite.internal.pagemem.PageIdAllocator.FLAG_IDX;
-import static org.apache.ignite.internal.processors.cache.database.tree.io.PageIO.getPageId;
-
-/**
- * Striped doubly-linked list of page IDs optionally organized in buckets.
- */
-public abstract class PagesList extends DataStructure {
- /** */
- private static final int TRY_LOCK_ATTEMPTS =
- IgniteSystemProperties.getInteger("IGNITE_PAGES_LIST_TRY_LOCK_ATTEMPTS", 10);
-
- /** */
- private static final int MAX_STRIPES_PER_BUCKET =
- IgniteSystemProperties.getInteger("IGNITE_PAGES_LIST_STRIPES_PER_BUCKET",
- Math.min(8, Runtime.getRuntime().availableProcessors() * 2));
-
- /** */
- protected final AtomicLong[] bucketsSize;
-
- /** */
- protected volatile boolean changed;
-
- /** Page ID to store list metadata. */
- private final long metaPageId;
-
- /** Number of buckets. */
- private final int buckets;
-
- /** Name (for debug purposes). */
- protected final String name;
-
- /** */
- private final PageHandler<Void, Boolean> cutTail = new CutTail();
-
- /**
- *
- */
- private final class CutTail extends PageHandler<Void, Boolean> {
- @Override public Boolean run(
- int cacheId,
- long pageId,
- long page,
- long pageAddr,
- PageIO iox,
- Boolean walPlc,
- Void ignore,
- int bucket) throws IgniteCheckedException {
- assert getPageId(pageAddr) == pageId;
-
- PagesListNodeIO io = (PagesListNodeIO)iox;
-
- long tailId = io.getNextId(pageAddr);
-
- assert tailId != 0;
-
- io.setNextId(pageAddr, 0L);
-
- if (needWalDeltaRecord(pageId, page, walPlc))
- wal.log(new PagesListSetNextRecord(cacheId, pageId, 0L));
-
- updateTail(bucket, tailId, pageId);
-
- return TRUE;
- }
- }
-
- /**
- * @param cacheId Cache ID.
- * @param name Name (for debug purpose).
- * @param pageMem Page memory.
- * @param buckets Number of buckets.
- * @param wal Write ahead log manager.
- * @param metaPageId Metadata page ID.
- */
- protected PagesList(
- int cacheId,
- String name,
- PageMemory pageMem,
- int buckets,
- IgniteWriteAheadLogManager wal,
- long metaPageId
- ) {
- super(cacheId, pageMem, wal);
-
- this.name = name;
- this.buckets = buckets;
- this.metaPageId = metaPageId;
-
- bucketsSize = new AtomicLong[buckets];
-
- for (int i = 0; i < buckets; i++)
- bucketsSize[i] = new AtomicLong();
- }
-
- /**
- * @param metaPageId Metadata page ID.
- * @param initNew {@code True} if new list if created, {@code false} if should be initialized from metadata.
- * @throws IgniteCheckedException If failed.
- */
- protected final void init(long metaPageId, boolean initNew) throws IgniteCheckedException {
- if (metaPageId != 0L) {
- if (initNew) {
- init(metaPageId, PagesListMetaIO.VERSIONS.latest());
- }
- else {
- Map<Integer, GridLongList> bucketsData = new HashMap<>();
-
- long nextId = metaPageId;
-
- while (nextId != 0) {
- final long pageId = nextId;
- final long page = acquirePage(pageId);
-
- try {
- long pageAddr = readLock(pageId, page); // No concurrent recycling on init.
-
- assert pageAddr != 0L;
-
- try {
- PagesListMetaIO io = PagesListMetaIO.VERSIONS.forPage(pageAddr);
-
- io.getBucketsData(pageAddr, bucketsData);
-
- nextId = io.getNextMetaPageId(pageAddr);
-
- assert nextId != pageId :
- "Loop detected [next=" + U.hexLong(nextId) + ", cur=" + U.hexLong(pageId) + ']';
-
-
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- for (Map.Entry<Integer, GridLongList> e : bucketsData.entrySet()) {
- int bucket = e.getKey();
- long bucketSize = 0;
-
- Stripe[] old = getBucket(bucket);
- assert old == null;
-
- long[] upd = e.getValue().array();
-
- Stripe[] tails = new Stripe[upd.length];
-
- for (int i = 0; i < upd.length; i++) {
- long tailId = upd[i];
-
- long prevId = tailId;
- int cnt = 0;
-
- while (prevId != 0L) {
- final long pageId = prevId;
- final long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page);
-
- assert pageAddr != 0L;
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
-
- cnt += io.getCount(pageAddr);
- prevId = io.getPreviousId(pageAddr);
-
- // In reuse bucket the page itself can be used as a free page.
- if (isReuseBucket(bucket) && prevId != 0L)
- cnt++;
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- Stripe stripe = new Stripe(tailId, cnt == 0);
- tails[i] = stripe;
- bucketSize += cnt;
- }
-
- boolean ok = casBucket(bucket, null, tails);
- assert ok;
-
- bucketsSize[bucket].set(bucketSize);
-
- changed = true;
- }
- }
- }
- }
-
- /**
- * @throws IgniteCheckedException If failed.
- */
- public void saveMetadata() throws IgniteCheckedException {
- assert metaPageId != 0;
-
- long curId = 0L;
- long curPage = 0L;
- long curAddr = 0L;
-
- PagesListMetaIO curIo = null;
-
- long nextPageId = metaPageId;
-
- if (!changed)
- return;
-
- try {
- for (int bucket = 0; bucket < buckets; bucket++) {
- Stripe[] tails = getBucket(bucket);
-
- if (tails != null) {
- int tailIdx = 0;
-
- while (tailIdx < tails.length) {
- int written = curPage != 0L ? curIo.addTails(pageMem.pageSize(), curAddr, bucket, tails, tailIdx) : 0;
-
- if (written == 0) {
- if (nextPageId == 0L) {
- nextPageId = allocatePageNoReuse();
-
- if (curPage != 0L) {
- curIo.setNextMetaPageId(curAddr, nextPageId);
-
- releaseAndClose(curId, curPage, curAddr);
- }
-
- curId = nextPageId;
- curPage = acquirePage(curId);
- curAddr = writeLock(curId, curPage);
-
- curIo = PagesListMetaIO.VERSIONS.latest();
-
- curIo.initNewPage(curAddr, curId, pageSize());
- }
- else {
- releaseAndClose(curId, curPage, curAddr);
-
- curId = nextPageId;
- curPage = acquirePage(curId);
- curAddr = writeLock(curId, curPage);
-
- curIo = PagesListMetaIO.VERSIONS.forPage(curAddr);
-
- curIo.resetCount(curAddr);
- }
-
- nextPageId = curIo.getNextMetaPageId(curAddr);
- }
- else
- tailIdx += written;
- }
- }
- }
- }
- finally {
- releaseAndClose(curId, curPage, curAddr);
- }
-
- while (nextPageId != 0L) {
- long pageId = nextPageId;
-
- long page = acquirePage(pageId);
- try {
- long pageAddr = writeLock(pageId, page);
-
- try {
- PagesListMetaIO io = PagesListMetaIO.VERSIONS.forPage(pageAddr);
-
- io.resetCount(pageAddr);
-
- if (needWalDeltaRecord(pageId, page, null))
- wal.log(new PageListMetaResetCountRecord(cacheId, pageId));
-
- nextPageId = io.getNextMetaPageId(pageAddr);
- }
- finally {
- writeUnlock(pageId, page, pageAddr, true);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- changed = false;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page absolute pointer.
- * @param pageAddr Page address.
- * @throws IgniteCheckedException If failed.
- */
- private void releaseAndClose(long pageId, long page, long pageAddr) throws IgniteCheckedException {
- if (page != 0L) {
- try {
- // No special WAL record because we most likely changed the whole page.
- writeUnlock(pageId, page, pageAddr, TRUE, true);
- }
- finally {
- releasePage(pageId, page);
- }
- }
- }
-
- /**
- * @param bucket Bucket index.
- * @return Bucket.
- */
- protected abstract Stripe[] getBucket(int bucket);
-
- /**
- * @param bucket Bucket index.
- * @param exp Expected bucket.
- * @param upd Updated bucket.
- * @return {@code true} If succeeded.
- */
- protected abstract boolean casBucket(int bucket, Stripe[] exp, Stripe[] upd);
-
- /**
- * @param bucket Bucket index.
- * @return {@code true} If it is a reuse bucket.
- */
- protected abstract boolean isReuseBucket(int bucket);
-
- /**
- * @param io IO.
- * @param prevId Previous page ID.
- * @param prev Previous page buffer.
- * @param nextId Next page ID.
- * @param next Next page buffer.
- */
- private void setupNextPage(PagesListNodeIO io, long prevId, long prev, long nextId, long next) {
- assert io.getNextId(prev) == 0L;
-
- io.initNewPage(next, nextId, pageSize());
- io.setPreviousId(next, prevId);
-
- io.setNextId(prev, nextId);
- }
-
- /**
- * Adds stripe to the given bucket.
- *
- * @param bucket Bucket.
- * @param reuse {@code True} if possible to use reuse list.
- * @throws IgniteCheckedException If failed.
- * @return Tail page ID.
- */
- private Stripe addStripe(int bucket, boolean reuse) throws IgniteCheckedException {
- long pageId = reuse ? allocatePage(null) : allocatePageNoReuse();
-
- init(pageId, PagesListNodeIO.VERSIONS.latest());
-
- Stripe stripe = new Stripe(pageId, true);
-
- for (;;) {
- Stripe[] old = getBucket(bucket);
- Stripe[] upd;
-
- if (old != null) {
- int len = old.length;
-
- upd = Arrays.copyOf(old, len + 1);
-
- upd[len] = stripe;
- }
- else
- upd = new Stripe[]{stripe};
-
- if (casBucket(bucket, old, upd))
- return stripe;
- }
- }
-
- /**
- * @param bucket Bucket index.
- * @param oldTailId Old tail page ID to replace.
- * @param newTailId New tail page ID.
- * @return {@code True} if stripe was removed.
- */
- private boolean updateTail(int bucket, long oldTailId, long newTailId) {
- int idx = -1;
-
- for (;;) {
- Stripe[] tails = getBucket(bucket);
-
- // Tail must exist to be updated.
- assert !F.isEmpty(tails) : "Missing tails [bucket=" + bucket + ", tails=" + Arrays.toString(tails) +
- ", metaPage=" + U.hexLong(metaPageId) + ']';
-
- idx = findTailIndex(tails, oldTailId, idx);
-
- assert tails[idx].tailId == oldTailId;
-
- if (newTailId == 0L) {
- if (tails.length <= MAX_STRIPES_PER_BUCKET / 2) {
- tails[idx].empty = true;
-
- return false;
- }
-
- Stripe[] newTails;
-
- if (tails.length != 1)
- newTails = GridArrays.remove(tails, idx);
- else
- newTails = null; // Drop the bucket completely.
-
- if (casBucket(bucket, tails, newTails))
- return true;
- }
- else {
- // It is safe to assign new tail since we do it only when write lock on tail is held.
- tails[idx].tailId = newTailId;
-
- return true;
- }
- }
- }
-
- /**
- * @param tails Tails.
- * @param tailId Tail ID to find.
- * @param expIdx Expected index.
- * @return First found index of the given tail ID.
- */
- private static int findTailIndex(Stripe[] tails, long tailId, int expIdx) {
- if (expIdx != -1 && tails.length > expIdx && tails[expIdx].tailId == tailId)
- return expIdx;
-
- for (int i = 0; i < tails.length; i++) {
- if (tails[i].tailId == tailId)
- return i;
- }
-
- throw new IllegalStateException("Tail not found: " + tailId);
- }
-
- /**
- * @param bucket Bucket.
- * @return Page ID where the given page
- * @throws IgniteCheckedException If failed.
- */
- private Stripe getPageForPut(int bucket) throws IgniteCheckedException {
- Stripe[] tails = getBucket(bucket);
-
- if (tails == null)
- return addStripe(bucket, true);
-
- return randomTail(tails);
- }
-
- /**
- * @param tails Tails.
- * @return Random tail.
- */
- private static Stripe randomTail(Stripe[] tails) {
- int len = tails.length;
-
- assert len != 0;
-
- return tails[randomInt(len)];
- }
-
- /**
- * !!! For tests only, does not provide any correctness guarantees for concurrent access.
- *
- * @param bucket Bucket index.
- * @return Number of pages stored in this list.
- * @throws IgniteCheckedException If failed.
- */
- protected final long storedPagesCount(int bucket) throws IgniteCheckedException {
- long res = 0;
-
- Stripe[] tails = getBucket(bucket);
-
- if (tails != null) {
- for (Stripe tail : tails) {
- long tailId = tail.tailId;
-
- while (tailId != 0L) {
- final long pageId = tailId;
- final long page = acquirePage(pageId);
- try {
- long pageAddr = readLock(pageId, page);
-
- assert pageAddr != 0L;
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
-
- int cnt = io.getCount(pageAddr);
-
- assert cnt >= 0;
-
- res += cnt;
- tailId = io.getPreviousId(pageAddr);
-
- // In reuse bucket the page itself can be used as a free page.
- if (isReuseBucket(bucket) && tailId != 0L)
- res++;
- }
- finally {
- readUnlock(pageId, page, pageAddr);
- }
- }
- finally {
- releasePage(pageId, page);
- }
- }
- }
- }
-
- assert res == bucketsSize[bucket].get() : "Wrong bucket size counter [exp=" + res + ", cntr=" + bucketsSize[bucket].get() + ']';
-
- return res;
- }
-
- /**
- * @param bag Reuse bag.
- * @param dataId Data page ID.
- * @param dataPage Data page pointer.
- * @param dataAddr Data page address.
- * @param bucket Bucket.
- * @throws IgniteCheckedException If failed.
- */
- protected final void put(
- ReuseBag bag,
- final long dataId,
- final long dataPage,
- final long dataAddr,
- int bucket)
- throws IgniteCheckedException {
- assert bag == null ^ dataAddr == 0L;
-
- for (int lockAttempt = 0; ;) {
- Stripe stripe = getPageForPut(bucket);
-
- final long tailId = stripe.tailId;
- final long tailPage = acquirePage(tailId);
-
- try {
- long tailAddr = writeLockPage(tailId, tailPage, bucket, lockAttempt++); // Explicit check.
-
- if (tailAddr == 0L) {
- if (isReuseBucket(bucket) && lockAttempt == TRY_LOCK_ATTEMPTS)
- addStripeForReuseBucket(bucket);
-
- continue;
- }
-
- assert PageIO.getPageId(tailAddr) == tailId : "pageId = " + PageIO.getPageId(tailAddr) + ", tailId = " + tailId;
- assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE;
-
- boolean ok = false;
-
- try {
- PagesListNodeIO io = PageIO.getPageIO(tailAddr);
-
- ok = bag != null ?
- // Here we can always take pages from the bag to build our list.
- putReuseBag(tailId, tailPage, tailAddr, io, bag, bucket) :
- // Here we can use the data page to build list only if it is empty and
- // it is being put into reuse bucket. Usually this will be true, but there is
- // a case when there is no reuse bucket in the free list, but then deadlock
- // on node page allocation from separate reuse list is impossible.
- // If the data page is not empty it can not be put into reuse bucket and thus
- // the deadlock is impossible as well.
- putDataPage(tailId, tailPage, tailAddr, io, dataId, dataPage, dataAddr, bucket);
-
- if (ok) {
- stripe.empty = false;
-
- return;
- }
- }
- finally {
- writeUnlock(tailId, tailPage, tailAddr, ok);
- }
- }
- finally {
- releasePage(tailId, tailPage);
- }
- }
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param dataId Data page ID.
- * @param dataPage Data page pointer.
- * @param dataAddr Data page address.
- * @param bucket Bucket.
- * @return {@code true} If succeeded.
- * @throws IgniteCheckedException If failed.
- */
- private boolean putDataPage(
- final long pageId,
- final long page,
- final long pageAddr,
- PagesListNodeIO io,
- final long dataId,
- final long dataPage,
- final long dataAddr,
- int bucket
- ) throws IgniteCheckedException {
- if (io.getNextId(pageAddr) != 0L)
- return false; // Splitted.
-
- int idx = io.addPage(pageAddr, dataId, pageSize());
-
- if (idx == -1)
- handlePageFull(pageId, page, pageAddr, io, dataId, dataPage, dataAddr, bucket);
- else {
- incrementBucketSize(bucket);
-
- if (needWalDeltaRecord(pageId, page, null))
- wal.log(new PagesListAddPageRecord(cacheId, pageId, dataId));
-
- DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
- dataIO.setFreeListPageId(dataAddr, pageId);
-
- if (needWalDeltaRecord(dataId, dataPage, null))
- wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, pageId));
- }
-
- return true;
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param dataId Data page ID.
- * @param data Data page pointer.
- * @param dataAddr Data page address.
- * @param bucket Bucket index.
- * @throws IgniteCheckedException If failed.
- * */
- private void handlePageFull(
- final long pageId,
- final long page,
- final long pageAddr,
- PagesListNodeIO io,
- final long dataId,
- final long data,
- final long dataAddr,
- int bucket
- ) throws IgniteCheckedException {
- DataPageIO dataIO = DataPageIO.VERSIONS.forPage(dataAddr);
-
- // Attempt to add page failed: the node page is full.
- if (isReuseBucket(bucket)) {
- // If we are on the reuse bucket, we can not allocate new page, because it may cause deadlock.
- assert dataIO.isEmpty(dataAddr); // We can put only empty data pages to reuse bucket.
-
- // Change page type to index and add it as next node page to this list.
- long newDataId = PageIdUtils.changeType(dataId, FLAG_IDX);
-
- setupNextPage(io, pageId, pageAddr, newDataId, dataAddr);
-
- if (needWalDeltaRecord(pageId, page, null))
- wal.log(new PagesListSetNextRecord(cacheId, pageId, newDataId));
-
- if (needWalDeltaRecord(dataId, data, null))
- wal.log(new PagesListInitNewPageRecord(
- cacheId,
- dataId,
- io.getType(),
- io.getVersion(),
- newDataId,
- pageId, 0L));
-
- // In reuse bucket the page itself can be used as a free page.
- incrementBucketSize(bucket);
-
- updateTail(bucket, pageId, newDataId);
- }
- else {
- // Just allocate a new node page and add our data page there.
- final long nextId = allocatePage(null);
- final long nextPage = acquirePage(nextId);
-
- try {
- long nextPageAddr = writeLock(nextId, nextPage); // Newly allocated page.
-
- assert nextPageAddr != 0L;
-
- // Here we should never write full page, because it is known to be new.
- Boolean nextWalPlc = FALSE;
-
- try {
- setupNextPage(io, pageId, pageAddr, nextId, nextPageAddr);
-
- if (needWalDeltaRecord(pageId, page, null))
- wal.log(new PagesListSetNextRecord(cacheId, pageId, nextId));
-
- int idx = io.addPage(nextPageAddr, dataId, pageSize());
-
- if (needWalDeltaRecord(nextId, nextPage, nextWalPlc))
- wal.log(new PagesListInitNewPageRecord(
- cacheId,
- nextId,
- io.getType(),
- io.getVersion(),
- nextId,
- pageId,
- dataId
- ));
-
- assert idx != -1;
-
- dataIO.setFreeListPageId(dataAddr, nextId);
-
- if (needWalDeltaRecord(dataId, data, null))
- wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, nextId));
-
- incrementBucketSize(bucket);
-
- updateTail(bucket, pageId, nextId);
- }
- finally {
- writeUnlock(nextId, nextPage, nextPageAddr, nextWalPlc, true);
- }
- }
- finally {
- releasePage(nextId, nextPage);
- }
- }
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param bag Reuse bag.
- * @param bucket Bucket.
- * @return {@code true} If succeeded.
- * @throws IgniteCheckedException if failed.
- */
- @SuppressWarnings("ForLoopReplaceableByForEach")
- private boolean putReuseBag(
- final long pageId,
- final long page,
- final long pageAddr,
- PagesListNodeIO io,
- ReuseBag bag,
- int bucket
- ) throws IgniteCheckedException {
- if (io.getNextId(pageAddr) != 0L)
- return false; // Splitted.
-
- long nextId;
-
- long prevId = pageId;
- long prevPage = page;
- long prevAddr = pageAddr;
-
- Boolean walPlc = null;
-
- GridLongList locked = null; // TODO may be unlock right away and do not keep all these pages locked?
-
- try {
- while ((nextId = bag.pollFreePage()) != 0L) {
- int idx = io.addPage(prevAddr, nextId, pageSize());
-
- if (idx == -1) { // Attempt to add page failed: the node page is full.
-
- final long nextPage = acquirePage(nextId);
-
- try {
- long nextPageAddr = writeLock(nextId, nextPage); // Page from reuse bag can't be concurrently recycled.
-
- assert nextPageAddr != 0L;
-
- if (locked == null) {
- locked = new GridLongList(6);
- }
-
- locked.add(nextId);
- locked.add(nextPage);
- locked.add(nextPageAddr);
-
- setupNextPage(io, prevId, prevAddr, nextId, nextPageAddr);
-
- if (needWalDeltaRecord(prevId, prevPage, walPlc))
- wal.log(new PagesListSetNextRecord(cacheId, prevId, nextId));
-
- // Here we should never write full page, because it is known to be new.
- if (needWalDeltaRecord(nextId, nextPage, FALSE))
- wal.log(new PagesListInitNewPageRecord(
- cacheId,
- nextId,
- io.getType(),
- io.getVersion(),
- nextId,
- prevId,
- 0L
- ));
-
- // In reuse bucket the page itself can be used as a free page.
- if (isReuseBucket(bucket))
- incrementBucketSize(bucket);
-
- // Switch to this new page, which is now a part of our list
- // to add the rest of the bag to the new page.
- prevAddr = nextPageAddr;
- prevId = nextId;
- prevPage = nextPage;
- // Starting from tis point all wal records are written for reused pages from the bag.
- // This mean that we use delta records only.
- walPlc = FALSE;
- }
- finally {
- releasePage(nextId, nextPage);
- }
- }
- else {
- // TODO: use single WAL record for bag?
- if (needWalDeltaRecord(prevId, prevPage, walPlc))
- wal.log(new PagesListAddPageRecord(cacheId, prevId, nextId));
-
- incrementBucketSize(bucket);
- }
- }
- }
- finally {
- if (locked != null) {
- // We have to update our bucket with the new tail.
- updateTail(bucket, pageId, prevId);
-
- // Release write.
- for (int i = 0; i < locked.size(); i+=3) {
- writeUnlock(locked.get(i), locked.get(i + 1), locked.get(i + 2), FALSE, true);
- }
- }
- }
-
- return true;
- }
-
- /**
- * @param bucket Bucket index.
- * @return Page for take.
- */
- private Stripe getPageForTake(int bucket) {
- Stripe[] tails = getBucket(bucket);
-
- if (tails == null || bucketsSize[bucket].get() == 0)
- return null;
-
- int len = tails.length;
- int init = randomInt(len);
- int cur = init;
-
- while (true) {
- Stripe stripe = tails[cur];
-
- if (!stripe.empty)
- return stripe;
-
- if ((cur = (cur + 1) % len) == init)
- return null;
- }
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param bucket Bucket.
- * @param lockAttempt Lock attempts counter.
- * @return Page address if page is locked of {@code null} if can retry lock.
- * @throws IgniteCheckedException If failed.
- */
- private long writeLockPage(long pageId, long page, int bucket, int lockAttempt)
- throws IgniteCheckedException {
- long pageAddr = tryWriteLock(pageId, page);
-
- if (pageAddr != 0L)
- return pageAddr;
-
- if (lockAttempt == TRY_LOCK_ATTEMPTS) {
- Stripe[] stripes = getBucket(bucket);
-
- if (stripes == null || stripes.length < MAX_STRIPES_PER_BUCKET) {
- if (!isReuseBucket(bucket))
- addStripe(bucket, true);
-
- return 0L;
- }
- }
-
- return lockAttempt < TRY_LOCK_ATTEMPTS ? 0L : writeLock(pageId, page); // Must be explicitly checked further.
- }
-
- /**
- * @param bucket Bucket.
- * @throws IgniteCheckedException If failed.
- */
- private void addStripeForReuseBucket(int bucket) throws IgniteCheckedException {
- assert isReuseBucket(bucket);
-
- Stripe[] stripes = getBucket(bucket);
-
- if (stripes == null || stripes.length < MAX_STRIPES_PER_BUCKET)
- addStripe(bucket, false);
- }
-
- /**
- * @param bucket Bucket index.
- * @param initIoVers Optional IO to initialize page.
- * @return Removed page ID.
- * @throws IgniteCheckedException If failed.
- */
- protected final long takeEmptyPage(int bucket, @Nullable IOVersions initIoVers) throws IgniteCheckedException {
- for (int lockAttempt = 0; ;) {
- Stripe stripe = getPageForTake(bucket);
-
- if (stripe == null)
- return 0L;
-
- final long tailId = stripe.tailId;
- final long tailPage = acquirePage(tailId);
-
- try {
- long tailAddr = writeLockPage(tailId, tailPage, bucket, lockAttempt++); // Explicit check.
-
- if (tailAddr == 0L) {
- if (isReuseBucket(bucket) && lockAttempt == TRY_LOCK_ATTEMPTS)
- addStripeForReuseBucket(bucket);
-
- continue;
- }
-
- if (stripe.empty) {
- // Another thread took the last page.
- writeUnlock(tailId, tailPage, tailAddr, false);
-
- if (bucketsSize[bucket].get() > 0) {
- lockAttempt--; // Ignore current attempt.
-
- continue;
- }
- else
- return 0L;
- }
-
- assert PageIO.getPageId(tailAddr) == tailId : "tailId = " + tailId + ", tailPageId = " + PageIO.getPageId(tailAddr);
- assert PageIO.getType(tailAddr) == PageIO.T_PAGE_LIST_NODE;
-
- boolean dirty = false;
- long dataPageId;
- long recycleId = 0L;
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(tailAddr);
-
- if (io.getNextId(tailAddr) != 0) {
- // It is not a tail anymore, retry.
- continue;
- }
-
- long pageId = io.takeAnyPage(tailAddr);
-
- if (pageId != 0L) {
- decrementBucketSize(bucket);
-
- if (needWalDeltaRecord(tailId, tailPage, null))
- wal.log(new PagesListRemovePageRecord(cacheId, tailId, pageId));
-
- dirty = true;
-
- dataPageId = pageId;
-
- if (io.isEmpty(tailAddr)) {
- long prevId = io.getPreviousId(tailAddr);
-
- // If we got an empty page in non-reuse bucket, move it back to reuse list
- // to prevent empty page leak to data pages.
- if (!isReuseBucket(bucket)) {
- if (prevId != 0L) {
- Boolean ok = write(prevId, cutTail, null, bucket, FALSE);
-
- assert ok == TRUE : ok;
-
- recycleId = recyclePage(tailId, tailPage, tailAddr, null);
- }
- else
- stripe.empty = true;
- }
- else
- stripe.empty = prevId == 0L;
- }
- }
- else {
- // The tail page is empty, but stripe is not. It might
- // happen only if we are in reuse bucket and it has
- // a previous page, so, the current page can be collected
- assert isReuseBucket(bucket);
-
- long prevId = io.getPreviousId(tailAddr);
-
- assert prevId != 0L;
-
- Boolean ok = write(prevId, cutTail, bucket, FALSE);
-
- assert ok == TRUE : ok;
-
- decrementBucketSize(bucket);
-
- if (initIoVers != null) {
- dataPageId = PageIdUtils.changeType(tailId, FLAG_DATA);
-
- PageIO initIo = initIoVers.latest();
-
- initIo.initNewPage(tailAddr, dataPageId, pageSize());
-
- if (needWalDeltaRecord(tailId, tailPage, null)) {
- wal.log(new InitNewPageRecord(cacheId, tailId, initIo.getType(),
- initIo.getVersion(), dataPageId));
- }
- }
- else
- dataPageId = recyclePage(tailId, tailPage, tailAddr, null);
-
- dirty = true;
- }
-
- // If we do not have a previous page (we are at head), then we still can return
- // current page but we have to drop the whole stripe. Since it is a reuse bucket,
- // we will not do that, but just return 0L, because this may produce contention on
- // meta page.
- }
- finally {
- writeUnlock(tailId, tailPage, tailAddr, dirty);
- }
-
- // Put recycled page (if any) to the reuse bucket after tail is unlocked.
- if (recycleId != 0L) {
- assert !isReuseBucket(bucket);
-
- reuseList.addForRecycle(new SingletonReuseBag(recycleId));
- }
-
- return dataPageId;
- }
- finally {
- releasePage(tailId, tailPage);
- }
- }
- }
-
- /**
- * @param dataId Data page ID.
- * @param dataPage Data page pointer.
- * @param dataAddr Data page address.
- * @param dataIO Data page IO.
- * @param bucket Bucket index.
- * @throws IgniteCheckedException If failed.
- * @return {@code True} if page was removed.
- */
- protected final boolean removeDataPage(
- final long dataId,
- final long dataPage,
- final long dataAddr,
- DataPageIO dataIO,
- int bucket)
- throws IgniteCheckedException {
- final long pageId = dataIO.getFreeListPageId(dataAddr);
-
- assert pageId != 0;
-
- final long page = acquirePage(pageId);
- try {
- long nextId;
-
- long recycleId = 0L;
-
- long pageAddr = writeLock(pageId, page); // Explicit check.
-
- if (pageAddr == 0L)
- return false;
-
- boolean rmvd = false;
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
-
- rmvd = io.removePage(pageAddr, dataId);
-
- if (!rmvd)
- return false;
-
- decrementBucketSize(bucket);
-
- if (needWalDeltaRecord(pageId, page, null))
- wal.log(new PagesListRemovePageRecord(cacheId, pageId, dataId));
-
- // Reset free list page ID.
- dataIO.setFreeListPageId(dataAddr, 0L);
-
- if (needWalDeltaRecord(dataId, dataPage, null))
- wal.log(new DataPageSetFreeListPageRecord(cacheId, dataId, 0L));
-
- if (!io.isEmpty(pageAddr))
- return true; // In optimistic case we still have something in the page and can leave it as is.
-
- // If the page is empty, we have to try to drop it and link next and previous with each other.
- nextId = io.getNextId(pageAddr);
-
- // If there are no next page, then we can try to merge without releasing current write lock,
- // because if we will need to lock previous page, the locking order will be already correct.
- if (nextId == 0L) {
- long prevId = io.getPreviousId(pageAddr);
-
- recycleId = mergeNoNext(pageId, page, pageAddr, prevId, bucket);
- }
- }
- finally {
- writeUnlock(pageId, page, pageAddr, rmvd);
- }
-
- // Perform a fair merge after lock release (to have a correct locking order).
- if (nextId != 0L)
- recycleId = merge(pageId, page, nextId, bucket);
-
- if (recycleId != 0L)
- reuseList.addForRecycle(new SingletonReuseBag(recycleId));
-
- return true;
- }
- finally {
- releasePage(pageId, page);
- }
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param pageAddr Page address.
- * @param prevId Previous page ID.
- * @param bucket Bucket index.
- * @return Page ID to recycle.
- * @throws IgniteCheckedException If failed.
- */
- private long mergeNoNext(
- long pageId,
- long page,
- long pageAddr,
- long prevId,
- int bucket)
- throws IgniteCheckedException {
- // If we do not have a next page (we are tail) and we are on reuse bucket,
- // then we can leave as is as well, because it is normal to have an empty tail page here.
- if (isReuseBucket(bucket))
- return 0L;
-
- if (prevId != 0L) { // Cut tail if we have a previous page.
- Boolean ok = write(prevId, cutTail, null, bucket, FALSE);
-
- assert ok == TRUE: ok;
- }
- else {
- // If we don't have a previous, then we are tail page of free list, just drop the stripe.
- boolean rmvd = updateTail(bucket, pageId, 0L);
-
- if (!rmvd)
- return 0L;
- }
-
- return recyclePage(pageId, page, pageAddr, null);
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page pointer.
- * @param nextId Next page ID.
- * @param bucket Bucket index.
- * @return Page ID to recycle.
- * @throws IgniteCheckedException If failed.
- */
- private long merge(
- final long pageId,
- final long page,
- long nextId,
- int bucket)
- throws IgniteCheckedException {
- assert nextId != 0; // We should do mergeNoNext then.
-
- // Lock all the pages in correct order (from next to previous) and do the merge in retry loop.
- for (;;) {
- final long curId = nextId;
- final long curPage = curId == 0L ? 0L : acquirePage(curId);
- try {
- boolean write = false;
-
- final long curAddr = curPage == 0L ? 0L : writeLock(curId, curPage); // Explicit check.
- final long pageAddr = writeLock(pageId, page); // Explicit check.
-
- if (pageAddr == 0L) {
- if (curAddr != 0L) // Unlock next page if needed.
- writeUnlock(curId, curPage, curAddr, false);
-
- return 0L; // Someone has merged or taken our empty page concurrently. Nothing to do here.
- }
-
- try {
- PagesListNodeIO io = PagesListNodeIO.VERSIONS.forPage(pageAddr);
-
- if (!io.isEmpty(pageAddr))
- return 0L; // No need to merge anymore.
-
- // Check if we see a consistent state of the world.
- if (io.getNextId(pageAddr) == curId && (curId == 0L) == (curAddr == 0L)) {
- long recycleId = doMerge(pageId, page, pageAddr, io, curId, curPage, curAddr, bucket);
-
- write = true;
-
- return recycleId; // Done.
- }
-
- // Reread next page ID and go for retry.
- nextId = io.getNextId(pageAddr);
- }
- finally {
- if (curAddr != 0L)
- writeUnlock(curId, curPage, curAddr, write);
-
- writeUnlock(pageId, page, pageAddr, write);
- }
- }
- finally {
- if (curPage != 0L)
- releasePage(curId, curPage);
- }
- }
- }
-
- /**
- * @param pageId Page ID.
- * @param page Page absolute pointer.
- * @param pageAddr Page address.
- * @param io IO.
- * @param nextId Next page ID.
- * @param nextPage Next page absolute pointer.
- * @param nextAddr Next page address.
- * @param bucket Bucket index.
- * @return Page to recycle.
- * @throws IgniteCheckedException If failed.
- */
- private long doMerge(
- long pageId,
- long page,
- long pageAddr,
- PagesListNodeIO io,
- long nextId,
- long nextPage,
- long nextAddr,
- int bucket
- ) throws IgniteCheckedException {
- long prevId = io.getPreviousId(pageAddr);
-
- if (nextId == 0L)
- return mergeNoNext(pageId, page, pageAddr, prevId, bucket);
- else {
- // No one must be able to merge it while we keep a reference.
- assert getPageId(nextAddr) == nextId;
-
- if (prevId == 0L) { // No previous page: we are at head.
- // These references must be updated at the same time in write locks.
- assert PagesListNodeIO.VERSIONS.forPage(nextAddr).getPreviousId(nextAddr) == pageId;
-
- PagesListNodeIO nextIO = PagesListNodeIO.VERSIONS.forPage(nextAddr);
- nextIO.setPreviousId(nextAddr, 0);
-
- if (needWalDeltaRecord(nextId, nextPage, null))
- wal.log(new PagesListSetPreviousRecord(cacheId, nextId, 0L));
- }
- else // Do a fair merge: link previous and next to each other.
- fairMerge(prevId, pageId, nextId, nextPage, nextAddr);
-
- return recyclePage(pageId, page, pageAddr, null);
- }
- }
-
- /**
- * Link previous and next to each other.
- * @param prevId Previous Previous page ID.
- * @param pageId Page ID.
- * @param nextId Next page ID.
- * @param nextPage Next page absolute pointer.
- * @param nextAddr Next page address.
- * @throws IgniteCheckedException If failed.
- */
- private void fairMerge(
- final long prevId,
- long pageId,
- long nextId,
- long nextPage,
- long nextAddr)
- throws IgniteCheckedException {
- long prevPage = acquirePage(prevId);
-
- try {
- final long prevAddr = writeLock(prevId, prevPage); // No check, we keep a reference.
- assert prevAddr != 0L;
- try {
- PagesListNodeIO prevIO = PagesListNodeIO.VERSIONS.forPage(prevAddr);
- PagesListNodeIO nextIO = PagesListNodeIO.VERSIONS.forPage(nextAddr);
-
- // These references must be updated at the same time in write locks.
- assert prevIO.getNextId(prevAddr) == pageId;
- assert nextIO.getPreviousId(nextAddr) == pageId;
-
- prevIO.setNextId(prevAddr, nextId);
-
- if (needWalDeltaRecord(prevId, prevPage, null))
- wal.log(new PagesListSetNextRecord(cacheId, prevId, nextId));
-
- nextIO.setPreviousId(nextAddr, prevId);
-
- if (needWalDeltaRecord(nextId, nextPage, null))
- wal.log(new PagesListSetPreviousRecord(cacheId, nextId, prevId));
- }
- finally {
- writeUnlock(prevId, prevPage, prevAddr, true);
- }
- }
- finally {
- releasePage(prevId, prevPage);
- }
- }
-
- /**
- * Increments bucket size and updates changed flag.
- *
- * @param bucket Bucket number.
- */
- private void incrementBucketSize(int bucket) {
- bucketsSize[bucket].incrementAndGet();
-
- // Ok to have a race here, see the field javadoc.
- if (!changed)
- changed = true;
- }
-
- /**
- * Increments bucket size and updates changed flag.
- *
- * @param bucket Bucket number.
- */
- private void decrementBucketSize(int bucket) {
- bucketsSize[bucket].decrementAndGet();
-
- // Ok to have a race here, see the field javadoc.
- if (!changed)
- changed = true;
- }
-
- /**
- * Singleton reuse bag.
- */
- private static final class SingletonReuseBag implements ReuseBag {
- /** */
- long pageId;
-
- /**
- * @param pageId Page ID.
- */
- SingletonReuseBag(long pageId) {
- this.pageId = pageId;
- }
-
- /** {@inheritDoc} */
- @Override public void addFreePage(long pageId) {
- throw new IllegalStateException("Should never be called.");
- }
-
- /** {@inheritDoc} */
- @Override public long pollFreePage() {
- long res = pageId;
-
- pageId = 0L;
-
- return res;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(SingletonReuseBag.class, this, "pageId", U.hexLong(pageId));
- }
- }
-
- /**
- *
- */
- public static final class Stripe {
- /** */
- public volatile long tailId;
-
- /** */
- volatile boolean empty;
-
- /**
- * @param tailId Tail ID.
- * @param empty Empty flag.
- */
- Stripe(long tailId, boolean empty) {
- this.tailId = tailId;
- this.empty = empty;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListMetaIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListMetaIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListMetaIO.java
deleted file mode 100644
index 6bd50a3..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListMetaIO.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.freelist.io;
-
-import java.util.Map;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.freelist.PagesList;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.util.GridLongList;
-
-/**
- *
- */
-public class PagesListMetaIO extends PageIO {
- /** */
- private static final int CNT_OFF = COMMON_HEADER_END;
-
- /** */
- private static final int NEXT_META_PAGE_OFF = CNT_OFF + 2;
-
- /** */
- private static final int ITEMS_OFF = NEXT_META_PAGE_OFF + 8;
-
- /** */
- private static final int ITEM_SIZE = 10;
-
- /** */
- public static final IOVersions<PagesListMetaIO> VERSIONS = new IOVersions<>(
- new PagesListMetaIO(1)
- );
-
- /**
- * @param ver Page format version.
- */
- private PagesListMetaIO(int ver) {
- super(T_PAGE_LIST_META, ver);
- }
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setCount(pageAddr, 0);
- setNextMetaPageId(pageAddr, 0L);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Stored items count.
- */
- private int getCount(long pageAddr) {
- return PageUtils.getShort(pageAddr, CNT_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param cnt Stored items count.
- */
- private void setCount(long pageAddr, int cnt) {
- assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
-
- PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Next meta page ID.
- */
- public long getNextMetaPageId(long pageAddr) {
- return PageUtils.getLong(pageAddr, NEXT_META_PAGE_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param metaPageId Next meta page ID.
- */
- public void setNextMetaPageId(long pageAddr, long metaPageId) {
- PageUtils.putLong(pageAddr, NEXT_META_PAGE_OFF, metaPageId);
- }
-
- /**
- * @param pageAddr Page address.
- */
- public void resetCount(long pageAddr) {
- setCount(pageAddr, 0);
- }
-
- /**
- * @param pageSize Page size.
- * @param pageAddr Page address.
- * @param bucket Bucket number.
- * @param tails Tails.
- * @param tailsOff Tails offset.
- * @return Number of items written.
- */
- public int addTails(int pageSize, long pageAddr, int bucket, PagesList.Stripe[] tails, int tailsOff) {
- assert bucket >= 0 && bucket <= Short.MAX_VALUE : bucket;
-
- int cnt = getCount(pageAddr);
- int cap = getCapacity(pageSize, pageAddr);
-
- if (cnt == cap)
- return 0;
-
- int off = offset(cnt);
-
- int write = Math.min(cap - cnt, tails.length - tailsOff);
-
- for (int i = 0; i < write; i++) {
- PageUtils.putShort(pageAddr, off, (short)bucket);
- PageUtils.putLong(pageAddr, off + 2, tails[tailsOff].tailId);
-
- tailsOff++;
-
- off += ITEM_SIZE;
- }
-
- setCount(pageAddr, cnt + write);
-
- return write;
- }
-
- /**
- * @param pageAddr Page address.
- * @param res Results map.
- */
- public void getBucketsData(long pageAddr, Map<Integer, GridLongList> res) {
- int cnt = getCount(pageAddr);
-
- assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
-
- if (cnt == 0)
- return;
-
- int off = offset(0);
-
- for (int i = 0; i < cnt; i++) {
- Integer bucket = (int)PageUtils.getShort(pageAddr, off);
- assert bucket >= 0 && bucket <= Short.MAX_VALUE : bucket;
-
- long tailId = PageUtils.getLong(pageAddr, off + 2);
- assert tailId != 0;
-
- GridLongList list = res.get(bucket);
-
- if (list == null)
- res.put(bucket, list = new GridLongList());
-
- list.add(tailId);
-
- off += ITEM_SIZE;
- }
- }
-
- /**
- * @param pageAddr Page address.
- * @return Maximum number of items which can be stored in buffer.
- */
- private int getCapacity(int pageSize, long pageAddr) {
- return (pageSize - ITEMS_OFF) / ITEM_SIZE;
- }
-
- /**
- * @param idx Item index.
- * @return Item offset.
- */
- private int offset(int idx) {
- return ITEMS_OFF + ITEM_SIZE * idx;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListNodeIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListNodeIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListNodeIO.java
deleted file mode 100644
index 6bd0532..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/io/PagesListNodeIO.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.freelist.io;
-
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-
-import static org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler.copyMemory;
-
-/**
- * TODO optimize: now we have slow {@link #removePage(long, long)}
- */
-public class PagesListNodeIO extends PageIO {
- /** */
- public static final IOVersions<PagesListNodeIO> VERSIONS = new IOVersions<>(
- new PagesListNodeIO(1)
- );
-
- /** */
- private static final int PREV_PAGE_ID_OFF = COMMON_HEADER_END;
-
- /** */
- private static final int NEXT_PAGE_ID_OFF = PREV_PAGE_ID_OFF + 8;
-
- /** */
- private static final int CNT_OFF = NEXT_PAGE_ID_OFF + 8;
-
- /** */
- private static final int PAGE_IDS_OFF = CNT_OFF + 2;
-
- /**
- * @param ver Page format version.
- */
- protected PagesListNodeIO(int ver) {
- super(T_PAGE_LIST_NODE, ver);
- }
-
- /** {@inheritDoc} */
- @Override public void initNewPage(long pageAddr, long pageId, int pageSize) {
- super.initNewPage(pageAddr, pageId, pageSize);
-
- setEmpty(pageAddr);
-
- setPreviousId(pageAddr, 0L);
- setNextId(pageAddr, 0L);
- }
-
- /**
- * @param pageAddr Page address.
- */
- private void setEmpty(long pageAddr) {
- setCount(pageAddr, 0);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Next page ID.
- */
- public long getNextId(long pageAddr) {
- return PageUtils.getLong(pageAddr, NEXT_PAGE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param nextId Next page ID.
- */
- public void setNextId(long pageAddr, long nextId) {
- PageUtils.putLong(pageAddr, NEXT_PAGE_ID_OFF, nextId);
- }
-
- /**
- * @param pageAddr Page address.
- * @return Previous page ID.
- */
- public long getPreviousId(long pageAddr) {
- return PageUtils.getLong(pageAddr, PREV_PAGE_ID_OFF);
- }
-
- /**
- * @param pageAddr Page address.
- * @param prevId Previous page ID.
- */
- public void setPreviousId(long pageAddr, long prevId) {
- PageUtils.putLong(pageAddr, PREV_PAGE_ID_OFF, prevId);
- }
-
- /**
- * Gets total count of entries in this page. Does not change the buffer state.
- *
- * @param pageAddr Page address to get count from.
- * @return Total number of entries.
- */
- public int getCount(long pageAddr) {
- return PageUtils.getShort(pageAddr, CNT_OFF);
- }
-
- /**
- * Sets total count of entries in this page. Does not change the buffer state.
- *
- * @param pageAddr Page address to write to.
- * @param cnt Count.
- */
- private void setCount(long pageAddr, int cnt) {
- assert cnt >= 0 && cnt <= Short.MAX_VALUE : cnt;
-
- PageUtils.putShort(pageAddr, CNT_OFF, (short)cnt);
- }
-
- /**
- * Gets capacity of this page in items.
- *
- * @param pageSize Page size.
- * @return Capacity of this page in items.
- */
- private int getCapacity(int pageSize) {
- return (pageSize - PAGE_IDS_OFF) >>> 3; // /8
- }
-
- /**
- * @param idx Item index.
- * @return Item offset.
- */
- private int offset(int idx) {
- return PAGE_IDS_OFF + 8 * idx;
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Item index.
- * @return Item at the given index.
- */
- private long getAt(long pageAddr, int idx) {
- return PageUtils.getLong(pageAddr, offset(idx));
- }
-
- /**
- * @param pageAddr Page address.
- * @param idx Item index.
- * @param pageId Item value to write.
- */
- private void setAt(long pageAddr, int idx, long pageId) {
- PageUtils.putLong(pageAddr, offset(idx), pageId);
- }
-
- /**
- * Adds page to the end of pages list.
- *
- * @param pageAddr Page address.
- * @param pageId Page ID.
- * @param pageSize Page size.
- * @return Total number of items in this page.
- */
- public int addPage(long pageAddr, long pageId, int pageSize) {
- int cnt = getCount(pageAddr);
-
- if (cnt == getCapacity(pageSize))
- return -1;
-
- setAt(pageAddr, cnt, pageId);
- setCount(pageAddr, cnt + 1);
-
- return cnt;
- }
-
- /**
- * Removes any page from the pages list.
- *
- * @param pageAddr Page address.
- * @return Removed page ID.
- */
- public long takeAnyPage(long pageAddr) {
- int cnt = getCount(pageAddr);
-
- if (cnt == 0)
- return 0L;
-
- setCount(pageAddr, --cnt);
-
- return getAt(pageAddr, cnt);
- }
-
- /**
- * Removes the given page ID from the pages list.
- *
- * @param pageAddr Page address.
- * @param dataPageId Page ID to remove.
- * @return {@code true} if page was in the list and was removed, {@code false} otherwise.
- */
- public boolean removePage(long pageAddr, long dataPageId) {
- assert dataPageId != 0;
-
- int cnt = getCount(pageAddr);
-
- for (int i = 0; i < cnt; i++) {
- if (getAt(pageAddr, i) == dataPageId) {
- if (i != cnt - 1)
- copyMemory(pageAddr, pageAddr, offset(i + 1), offset(i), 8 * (cnt - i - 1));
-
- setCount(pageAddr, cnt - 1);
-
- return true;
- }
- }
-
- return false;
- }
-
- /**
- * @param pageAddr Page address.
- * @return {@code True} if there are no items in this page.
- */
- public boolean isEmpty(long pageAddr) {
- return getCount(pageAddr) == 0;
- }
-}
[27/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java
new file mode 100644
index 0000000..80c375e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/SegmentEofException.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal;
+
+import org.apache.ignite.IgniteCheckedException;
+
+/**
+ * This exception is thrown either when we reach the end of file of WAL segment, or when we encounter
+ * a record with type equal to {@code 0}.
+ */
+public class SegmentEofException extends IgniteCheckedException {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * @param msg Message.
+ * @param cause Cause.
+ */
+ public SegmentEofException(String msg, Throwable cause) {
+ super(msg, cause, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/IgniteDataIntegrityViolationException.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/IgniteDataIntegrityViolationException.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/IgniteDataIntegrityViolationException.java
new file mode 100644
index 0000000..b4d0ae0
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/IgniteDataIntegrityViolationException.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.crc;
+
+import org.apache.ignite.IgniteException;
+
+/**
+ * Will be thrown if data integrity violation is found
+ */
+public class IgniteDataIntegrityViolationException extends IgniteException {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /**
+ * Default constructor.
+ */
+ public IgniteDataIntegrityViolationException() {
+ }
+
+ /**
+ * @param msg Message.
+ */
+ public IgniteDataIntegrityViolationException(String msg) {
+ super(msg);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java
new file mode 100644
index 0000000..6bd4a35
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/crc/PureJavaCrc32.java
@@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.crc;
+
+
+import java.nio.ByteBuffer;
+
+/**
+ * This class was taken from Hadoop org.apache.hadoop.util.PureJavaCrc32
+ *
+ * A pure-java implementation of the CRC32 checksum that uses the same polynomial as the built-in native CRC32.
+ *
+ * This is to avoid the JNI overhead for certain uses of Checksumming where many small pieces of data are checksummed in
+ * succession.
+ *
+ * The current version is ~10x to 1.8x as fast as Sun's native java.util.zip.CRC32 in Java 1.6
+ */
+public class PureJavaCrc32 {
+ /**
+ * the current CRC value
+ */
+ private int crc;
+
+ /**
+ * Create a new PureJavaCrc32 object.
+ */
+ public PureJavaCrc32() {
+ reset();
+ }
+
+ /**
+ * return crc32 value
+ */
+ public int getValue() {
+ return crc;
+ }
+
+ /**
+ * preparation for further calculations
+ */
+ public void reset() {
+ crc = 0xffffffff;
+ }
+
+ /**
+ * @param b B.
+ * @param len Length.
+ */
+ public void update(final ByteBuffer b, final int len) {
+ crc = calcCrc32(crc, b, len);
+ }
+
+ /**
+ * @param b B.
+ */
+ final public void update(int b) {
+ crc = (crc >>> 8) ^ T[(((crc ^ b) << 24) >>> 24)];
+ }
+
+
+ /**
+ * Calculates CRC32 checksum. This method will move buffer's position to {@code len} bytes forward.
+ *
+ * @param b B.
+ * @param len Length.
+ */
+ public static int calcCrc32(ByteBuffer b, final int len) {
+ return calcCrc32(0xffffffff, b, len);
+ }
+
+ /**
+ * @param crc0 Crc 0.
+ * @param b B.
+ * @param len Length.
+ */
+ private static int calcCrc32(int crc0, ByteBuffer b, final int len) {
+ int crc = crc0;
+
+ final int remainder = len & 0x7;
+ int i = 0;
+ for (; i < len - remainder; i += 8) {
+ final int x = crc
+ ^ ((((b.get() << 24) >>> 24) + ((b.get() << 24) >>> 16))
+ + (((b.get() << 24) >>> 8) + (b.get() << 24)));
+
+ crc = ((T[((x << 24) >>> 24) + 0x700] ^ T[((x << 16) >>> 24) + 0x600])
+ ^ (T[((x << 8) >>> 24) + 0x500] ^ T[(x >>> 24) + 0x400]))
+ ^ ((T[((b.get() << 24) >>> 24) + 0x300] ^ T[((b.get() << 24) >>> 24) + 0x200])
+ ^ (T[((b.get() << 24) >>> 24) + 0x100] ^ T[((b.get() << 24) >>> 24)]));
+ }
+
+ /* loop unroll - duff's device style */
+ switch (remainder) {
+ case 7:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ case 6:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ case 5:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ case 4:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ case 3:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ case 2:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ case 1:
+ crc = (crc >>> 8) ^ T[((crc ^ b.get()) << 24) >>> 24];
+ default:
+ /* nothing */
+ }
+
+ return crc;
+ }
+
+ /**
+ * CRC-32 lookup tables generated by the polynomial 0xEDB88320.
+ */
+ private static final int[] T = new int[]{
+ /* T8_0 */
+ 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA,
+ 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3,
+ 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
+ 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91,
+ 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE,
+ 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
+ 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,
+ 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5,
+ 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
+ 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B,
+ 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940,
+ 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
+ 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116,
+ 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F,
+ 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
+ 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D,
+ 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A,
+ 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
+ 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818,
+ 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01,
+ 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
+ 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457,
+ 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C,
+ 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
+ 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2,
+ 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB,
+ 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
+ 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9,
+ 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086,
+ 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
+ 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4,
+ 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD,
+ 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
+ 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683,
+ 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8,
+ 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
+ 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE,
+ 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7,
+ 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
+ 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5,
+ 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252,
+ 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
+ 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60,
+ 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79,
+ 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
+ 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F,
+ 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04,
+ 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
+ 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A,
+ 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713,
+ 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
+ 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21,
+ 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E,
+ 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
+ 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C,
+ 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45,
+ 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
+ 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB,
+ 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0,
+ 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
+ 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6,
+ 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF,
+ 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
+ 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D,
+ /* T8_1 */
+ 0x00000000, 0x191B3141, 0x32366282, 0x2B2D53C3,
+ 0x646CC504, 0x7D77F445, 0x565AA786, 0x4F4196C7,
+ 0xC8D98A08, 0xD1C2BB49, 0xFAEFE88A, 0xE3F4D9CB,
+ 0xACB54F0C, 0xB5AE7E4D, 0x9E832D8E, 0x87981CCF,
+ 0x4AC21251, 0x53D92310, 0x78F470D3, 0x61EF4192,
+ 0x2EAED755, 0x37B5E614, 0x1C98B5D7, 0x05838496,
+ 0x821B9859, 0x9B00A918, 0xB02DFADB, 0xA936CB9A,
+ 0xE6775D5D, 0xFF6C6C1C, 0xD4413FDF, 0xCD5A0E9E,
+ 0x958424A2, 0x8C9F15E3, 0xA7B24620, 0xBEA97761,
+ 0xF1E8E1A6, 0xE8F3D0E7, 0xC3DE8324, 0xDAC5B265,
+ 0x5D5DAEAA, 0x44469FEB, 0x6F6BCC28, 0x7670FD69,
+ 0x39316BAE, 0x202A5AEF, 0x0B07092C, 0x121C386D,
+ 0xDF4636F3, 0xC65D07B2, 0xED705471, 0xF46B6530,
+ 0xBB2AF3F7, 0xA231C2B6, 0x891C9175, 0x9007A034,
+ 0x179FBCFB, 0x0E848DBA, 0x25A9DE79, 0x3CB2EF38,
+ 0x73F379FF, 0x6AE848BE, 0x41C51B7D, 0x58DE2A3C,
+ 0xF0794F05, 0xE9627E44, 0xC24F2D87, 0xDB541CC6,
+ 0x94158A01, 0x8D0EBB40, 0xA623E883, 0xBF38D9C2,
+ 0x38A0C50D, 0x21BBF44C, 0x0A96A78F, 0x138D96CE,
+ 0x5CCC0009, 0x45D73148, 0x6EFA628B, 0x77E153CA,
+ 0xBABB5D54, 0xA3A06C15, 0x888D3FD6, 0x91960E97,
+ 0xDED79850, 0xC7CCA911, 0xECE1FAD2, 0xF5FACB93,
+ 0x7262D75C, 0x6B79E61D, 0x4054B5DE, 0x594F849F,
+ 0x160E1258, 0x0F152319, 0x243870DA, 0x3D23419B,
+ 0x65FD6BA7, 0x7CE65AE6, 0x57CB0925, 0x4ED03864,
+ 0x0191AEA3, 0x188A9FE2, 0x33A7CC21, 0x2ABCFD60,
+ 0xAD24E1AF, 0xB43FD0EE, 0x9F12832D, 0x8609B26C,
+ 0xC94824AB, 0xD05315EA, 0xFB7E4629, 0xE2657768,
+ 0x2F3F79F6, 0x362448B7, 0x1D091B74, 0x04122A35,
+ 0x4B53BCF2, 0x52488DB3, 0x7965DE70, 0x607EEF31,
+ 0xE7E6F3FE, 0xFEFDC2BF, 0xD5D0917C, 0xCCCBA03D,
+ 0x838A36FA, 0x9A9107BB, 0xB1BC5478, 0xA8A76539,
+ 0x3B83984B, 0x2298A90A, 0x09B5FAC9, 0x10AECB88,
+ 0x5FEF5D4F, 0x46F46C0E, 0x6DD93FCD, 0x74C20E8C,
+ 0xF35A1243, 0xEA412302, 0xC16C70C1, 0xD8774180,
+ 0x9736D747, 0x8E2DE606, 0xA500B5C5, 0xBC1B8484,
+ 0x71418A1A, 0x685ABB5B, 0x4377E898, 0x5A6CD9D9,
+ 0x152D4F1E, 0x0C367E5F, 0x271B2D9C, 0x3E001CDD,
+ 0xB9980012, 0xA0833153, 0x8BAE6290, 0x92B553D1,
+ 0xDDF4C516, 0xC4EFF457, 0xEFC2A794, 0xF6D996D5,
+ 0xAE07BCE9, 0xB71C8DA8, 0x9C31DE6B, 0x852AEF2A,
+ 0xCA6B79ED, 0xD37048AC, 0xF85D1B6F, 0xE1462A2E,
+ 0x66DE36E1, 0x7FC507A0, 0x54E85463, 0x4DF36522,
+ 0x02B2F3E5, 0x1BA9C2A4, 0x30849167, 0x299FA026,
+ 0xE4C5AEB8, 0xFDDE9FF9, 0xD6F3CC3A, 0xCFE8FD7B,
+ 0x80A96BBC, 0x99B25AFD, 0xB29F093E, 0xAB84387F,
+ 0x2C1C24B0, 0x350715F1, 0x1E2A4632, 0x07317773,
+ 0x4870E1B4, 0x516BD0F5, 0x7A468336, 0x635DB277,
+ 0xCBFAD74E, 0xD2E1E60F, 0xF9CCB5CC, 0xE0D7848D,
+ 0xAF96124A, 0xB68D230B, 0x9DA070C8, 0x84BB4189,
+ 0x03235D46, 0x1A386C07, 0x31153FC4, 0x280E0E85,
+ 0x674F9842, 0x7E54A903, 0x5579FAC0, 0x4C62CB81,
+ 0x8138C51F, 0x9823F45E, 0xB30EA79D, 0xAA1596DC,
+ 0xE554001B, 0xFC4F315A, 0xD7626299, 0xCE7953D8,
+ 0x49E14F17, 0x50FA7E56, 0x7BD72D95, 0x62CC1CD4,
+ 0x2D8D8A13, 0x3496BB52, 0x1FBBE891, 0x06A0D9D0,
+ 0x5E7EF3EC, 0x4765C2AD, 0x6C48916E, 0x7553A02F,
+ 0x3A1236E8, 0x230907A9, 0x0824546A, 0x113F652B,
+ 0x96A779E4, 0x8FBC48A5, 0xA4911B66, 0xBD8A2A27,
+ 0xF2CBBCE0, 0xEBD08DA1, 0xC0FDDE62, 0xD9E6EF23,
+ 0x14BCE1BD, 0x0DA7D0FC, 0x268A833F, 0x3F91B27E,
+ 0x70D024B9, 0x69CB15F8, 0x42E6463B, 0x5BFD777A,
+ 0xDC656BB5, 0xC57E5AF4, 0xEE530937, 0xF7483876,
+ 0xB809AEB1, 0xA1129FF0, 0x8A3FCC33, 0x9324FD72,
+ /* T8_2 */
+ 0x00000000, 0x01C26A37, 0x0384D46E, 0x0246BE59,
+ 0x0709A8DC, 0x06CBC2EB, 0x048D7CB2, 0x054F1685,
+ 0x0E1351B8, 0x0FD13B8F, 0x0D9785D6, 0x0C55EFE1,
+ 0x091AF964, 0x08D89353, 0x0A9E2D0A, 0x0B5C473D,
+ 0x1C26A370, 0x1DE4C947, 0x1FA2771E, 0x1E601D29,
+ 0x1B2F0BAC, 0x1AED619B, 0x18ABDFC2, 0x1969B5F5,
+ 0x1235F2C8, 0x13F798FF, 0x11B126A6, 0x10734C91,
+ 0x153C5A14, 0x14FE3023, 0x16B88E7A, 0x177AE44D,
+ 0x384D46E0, 0x398F2CD7, 0x3BC9928E, 0x3A0BF8B9,
+ 0x3F44EE3C, 0x3E86840B, 0x3CC03A52, 0x3D025065,
+ 0x365E1758, 0x379C7D6F, 0x35DAC336, 0x3418A901,
+ 0x3157BF84, 0x3095D5B3, 0x32D36BEA, 0x331101DD,
+ 0x246BE590, 0x25A98FA7, 0x27EF31FE, 0x262D5BC9,
+ 0x23624D4C, 0x22A0277B, 0x20E69922, 0x2124F315,
+ 0x2A78B428, 0x2BBADE1F, 0x29FC6046, 0x283E0A71,
+ 0x2D711CF4, 0x2CB376C3, 0x2EF5C89A, 0x2F37A2AD,
+ 0x709A8DC0, 0x7158E7F7, 0x731E59AE, 0x72DC3399,
+ 0x7793251C, 0x76514F2B, 0x7417F172, 0x75D59B45,
+ 0x7E89DC78, 0x7F4BB64F, 0x7D0D0816, 0x7CCF6221,
+ 0x798074A4, 0x78421E93, 0x7A04A0CA, 0x7BC6CAFD,
+ 0x6CBC2EB0, 0x6D7E4487, 0x6F38FADE, 0x6EFA90E9,
+ 0x6BB5866C, 0x6A77EC5B, 0x68315202, 0x69F33835,
+ 0x62AF7F08, 0x636D153F, 0x612BAB66, 0x60E9C151,
+ 0x65A6D7D4, 0x6464BDE3, 0x662203BA, 0x67E0698D,
+ 0x48D7CB20, 0x4915A117, 0x4B531F4E, 0x4A917579,
+ 0x4FDE63FC, 0x4E1C09CB, 0x4C5AB792, 0x4D98DDA5,
+ 0x46C49A98, 0x4706F0AF, 0x45404EF6, 0x448224C1,
+ 0x41CD3244, 0x400F5873, 0x4249E62A, 0x438B8C1D,
+ 0x54F16850, 0x55330267, 0x5775BC3E, 0x56B7D609,
+ 0x53F8C08C, 0x523AAABB, 0x507C14E2, 0x51BE7ED5,
+ 0x5AE239E8, 0x5B2053DF, 0x5966ED86, 0x58A487B1,
+ 0x5DEB9134, 0x5C29FB03, 0x5E6F455A, 0x5FAD2F6D,
+ 0xE1351B80, 0xE0F771B7, 0xE2B1CFEE, 0xE373A5D9,
+ 0xE63CB35C, 0xE7FED96B, 0xE5B86732, 0xE47A0D05,
+ 0xEF264A38, 0xEEE4200F, 0xECA29E56, 0xED60F461,
+ 0xE82FE2E4, 0xE9ED88D3, 0xEBAB368A, 0xEA695CBD,
+ 0xFD13B8F0, 0xFCD1D2C7, 0xFE976C9E, 0xFF5506A9,
+ 0xFA1A102C, 0xFBD87A1B, 0xF99EC442, 0xF85CAE75,
+ 0xF300E948, 0xF2C2837F, 0xF0843D26, 0xF1465711,
+ 0xF4094194, 0xF5CB2BA3, 0xF78D95FA, 0xF64FFFCD,
+ 0xD9785D60, 0xD8BA3757, 0xDAFC890E, 0xDB3EE339,
+ 0xDE71F5BC, 0xDFB39F8B, 0xDDF521D2, 0xDC374BE5,
+ 0xD76B0CD8, 0xD6A966EF, 0xD4EFD8B6, 0xD52DB281,
+ 0xD062A404, 0xD1A0CE33, 0xD3E6706A, 0xD2241A5D,
+ 0xC55EFE10, 0xC49C9427, 0xC6DA2A7E, 0xC7184049,
+ 0xC25756CC, 0xC3953CFB, 0xC1D382A2, 0xC011E895,
+ 0xCB4DAFA8, 0xCA8FC59F, 0xC8C97BC6, 0xC90B11F1,
+ 0xCC440774, 0xCD866D43, 0xCFC0D31A, 0xCE02B92D,
+ 0x91AF9640, 0x906DFC77, 0x922B422E, 0x93E92819,
+ 0x96A63E9C, 0x976454AB, 0x9522EAF2, 0x94E080C5,
+ 0x9FBCC7F8, 0x9E7EADCF, 0x9C381396, 0x9DFA79A1,
+ 0x98B56F24, 0x99770513, 0x9B31BB4A, 0x9AF3D17D,
+ 0x8D893530, 0x8C4B5F07, 0x8E0DE15E, 0x8FCF8B69,
+ 0x8A809DEC, 0x8B42F7DB, 0x89044982, 0x88C623B5,
+ 0x839A6488, 0x82580EBF, 0x801EB0E6, 0x81DCDAD1,
+ 0x8493CC54, 0x8551A663, 0x8717183A, 0x86D5720D,
+ 0xA9E2D0A0, 0xA820BA97, 0xAA6604CE, 0xABA46EF9,
+ 0xAEEB787C, 0xAF29124B, 0xAD6FAC12, 0xACADC625,
+ 0xA7F18118, 0xA633EB2F, 0xA4755576, 0xA5B73F41,
+ 0xA0F829C4, 0xA13A43F3, 0xA37CFDAA, 0xA2BE979D,
+ 0xB5C473D0, 0xB40619E7, 0xB640A7BE, 0xB782CD89,
+ 0xB2CDDB0C, 0xB30FB13B, 0xB1490F62, 0xB08B6555,
+ 0xBBD72268, 0xBA15485F, 0xB853F606, 0xB9919C31,
+ 0xBCDE8AB4, 0xBD1CE083, 0xBF5A5EDA, 0xBE9834ED,
+ /* T8_3 */
+ 0x00000000, 0xB8BC6765, 0xAA09C88B, 0x12B5AFEE,
+ 0x8F629757, 0x37DEF032, 0x256B5FDC, 0x9DD738B9,
+ 0xC5B428EF, 0x7D084F8A, 0x6FBDE064, 0xD7018701,
+ 0x4AD6BFB8, 0xF26AD8DD, 0xE0DF7733, 0x58631056,
+ 0x5019579F, 0xE8A530FA, 0xFA109F14, 0x42ACF871,
+ 0xDF7BC0C8, 0x67C7A7AD, 0x75720843, 0xCDCE6F26,
+ 0x95AD7F70, 0x2D111815, 0x3FA4B7FB, 0x8718D09E,
+ 0x1ACFE827, 0xA2738F42, 0xB0C620AC, 0x087A47C9,
+ 0xA032AF3E, 0x188EC85B, 0x0A3B67B5, 0xB28700D0,
+ 0x2F503869, 0x97EC5F0C, 0x8559F0E2, 0x3DE59787,
+ 0x658687D1, 0xDD3AE0B4, 0xCF8F4F5A, 0x7733283F,
+ 0xEAE41086, 0x525877E3, 0x40EDD80D, 0xF851BF68,
+ 0xF02BF8A1, 0x48979FC4, 0x5A22302A, 0xE29E574F,
+ 0x7F496FF6, 0xC7F50893, 0xD540A77D, 0x6DFCC018,
+ 0x359FD04E, 0x8D23B72B, 0x9F9618C5, 0x272A7FA0,
+ 0xBAFD4719, 0x0241207C, 0x10F48F92, 0xA848E8F7,
+ 0x9B14583D, 0x23A83F58, 0x311D90B6, 0x89A1F7D3,
+ 0x1476CF6A, 0xACCAA80F, 0xBE7F07E1, 0x06C36084,
+ 0x5EA070D2, 0xE61C17B7, 0xF4A9B859, 0x4C15DF3C,
+ 0xD1C2E785, 0x697E80E0, 0x7BCB2F0E, 0xC377486B,
+ 0xCB0D0FA2, 0x73B168C7, 0x6104C729, 0xD9B8A04C,
+ 0x446F98F5, 0xFCD3FF90, 0xEE66507E, 0x56DA371B,
+ 0x0EB9274D, 0xB6054028, 0xA4B0EFC6, 0x1C0C88A3,
+ 0x81DBB01A, 0x3967D77F, 0x2BD27891, 0x936E1FF4,
+ 0x3B26F703, 0x839A9066, 0x912F3F88, 0x299358ED,
+ 0xB4446054, 0x0CF80731, 0x1E4DA8DF, 0xA6F1CFBA,
+ 0xFE92DFEC, 0x462EB889, 0x549B1767, 0xEC277002,
+ 0x71F048BB, 0xC94C2FDE, 0xDBF98030, 0x6345E755,
+ 0x6B3FA09C, 0xD383C7F9, 0xC1366817, 0x798A0F72,
+ 0xE45D37CB, 0x5CE150AE, 0x4E54FF40, 0xF6E89825,
+ 0xAE8B8873, 0x1637EF16, 0x048240F8, 0xBC3E279D,
+ 0x21E91F24, 0x99557841, 0x8BE0D7AF, 0x335CB0CA,
+ 0xED59B63B, 0x55E5D15E, 0x47507EB0, 0xFFEC19D5,
+ 0x623B216C, 0xDA874609, 0xC832E9E7, 0x708E8E82,
+ 0x28ED9ED4, 0x9051F9B1, 0x82E4565F, 0x3A58313A,
+ 0xA78F0983, 0x1F336EE6, 0x0D86C108, 0xB53AA66D,
+ 0xBD40E1A4, 0x05FC86C1, 0x1749292F, 0xAFF54E4A,
+ 0x322276F3, 0x8A9E1196, 0x982BBE78, 0x2097D91D,
+ 0x78F4C94B, 0xC048AE2E, 0xD2FD01C0, 0x6A4166A5,
+ 0xF7965E1C, 0x4F2A3979, 0x5D9F9697, 0xE523F1F2,
+ 0x4D6B1905, 0xF5D77E60, 0xE762D18E, 0x5FDEB6EB,
+ 0xC2098E52, 0x7AB5E937, 0x680046D9, 0xD0BC21BC,
+ 0x88DF31EA, 0x3063568F, 0x22D6F961, 0x9A6A9E04,
+ 0x07BDA6BD, 0xBF01C1D8, 0xADB46E36, 0x15080953,
+ 0x1D724E9A, 0xA5CE29FF, 0xB77B8611, 0x0FC7E174,
+ 0x9210D9CD, 0x2AACBEA8, 0x38191146, 0x80A57623,
+ 0xD8C66675, 0x607A0110, 0x72CFAEFE, 0xCA73C99B,
+ 0x57A4F122, 0xEF189647, 0xFDAD39A9, 0x45115ECC,
+ 0x764DEE06, 0xCEF18963, 0xDC44268D, 0x64F841E8,
+ 0xF92F7951, 0x41931E34, 0x5326B1DA, 0xEB9AD6BF,
+ 0xB3F9C6E9, 0x0B45A18C, 0x19F00E62, 0xA14C6907,
+ 0x3C9B51BE, 0x842736DB, 0x96929935, 0x2E2EFE50,
+ 0x2654B999, 0x9EE8DEFC, 0x8C5D7112, 0x34E11677,
+ 0xA9362ECE, 0x118A49AB, 0x033FE645, 0xBB838120,
+ 0xE3E09176, 0x5B5CF613, 0x49E959FD, 0xF1553E98,
+ 0x6C820621, 0xD43E6144, 0xC68BCEAA, 0x7E37A9CF,
+ 0xD67F4138, 0x6EC3265D, 0x7C7689B3, 0xC4CAEED6,
+ 0x591DD66F, 0xE1A1B10A, 0xF3141EE4, 0x4BA87981,
+ 0x13CB69D7, 0xAB770EB2, 0xB9C2A15C, 0x017EC639,
+ 0x9CA9FE80, 0x241599E5, 0x36A0360B, 0x8E1C516E,
+ 0x866616A7, 0x3EDA71C2, 0x2C6FDE2C, 0x94D3B949,
+ 0x090481F0, 0xB1B8E695, 0xA30D497B, 0x1BB12E1E,
+ 0x43D23E48, 0xFB6E592D, 0xE9DBF6C3, 0x516791A6,
+ 0xCCB0A91F, 0x740CCE7A, 0x66B96194, 0xDE0506F1,
+ /* T8_4 */
+ 0x00000000, 0x3D6029B0, 0x7AC05360, 0x47A07AD0,
+ 0xF580A6C0, 0xC8E08F70, 0x8F40F5A0, 0xB220DC10,
+ 0x30704BC1, 0x0D106271, 0x4AB018A1, 0x77D03111,
+ 0xC5F0ED01, 0xF890C4B1, 0xBF30BE61, 0x825097D1,
+ 0x60E09782, 0x5D80BE32, 0x1A20C4E2, 0x2740ED52,
+ 0x95603142, 0xA80018F2, 0xEFA06222, 0xD2C04B92,
+ 0x5090DC43, 0x6DF0F5F3, 0x2A508F23, 0x1730A693,
+ 0xA5107A83, 0x98705333, 0xDFD029E3, 0xE2B00053,
+ 0xC1C12F04, 0xFCA106B4, 0xBB017C64, 0x866155D4,
+ 0x344189C4, 0x0921A074, 0x4E81DAA4, 0x73E1F314,
+ 0xF1B164C5, 0xCCD14D75, 0x8B7137A5, 0xB6111E15,
+ 0x0431C205, 0x3951EBB5, 0x7EF19165, 0x4391B8D5,
+ 0xA121B886, 0x9C419136, 0xDBE1EBE6, 0xE681C256,
+ 0x54A11E46, 0x69C137F6, 0x2E614D26, 0x13016496,
+ 0x9151F347, 0xAC31DAF7, 0xEB91A027, 0xD6F18997,
+ 0x64D15587, 0x59B17C37, 0x1E1106E7, 0x23712F57,
+ 0x58F35849, 0x659371F9, 0x22330B29, 0x1F532299,
+ 0xAD73FE89, 0x9013D739, 0xD7B3ADE9, 0xEAD38459,
+ 0x68831388, 0x55E33A38, 0x124340E8, 0x2F236958,
+ 0x9D03B548, 0xA0639CF8, 0xE7C3E628, 0xDAA3CF98,
+ 0x3813CFCB, 0x0573E67B, 0x42D39CAB, 0x7FB3B51B,
+ 0xCD93690B, 0xF0F340BB, 0xB7533A6B, 0x8A3313DB,
+ 0x0863840A, 0x3503ADBA, 0x72A3D76A, 0x4FC3FEDA,
+ 0xFDE322CA, 0xC0830B7A, 0x872371AA, 0xBA43581A,
+ 0x9932774D, 0xA4525EFD, 0xE3F2242D, 0xDE920D9D,
+ 0x6CB2D18D, 0x51D2F83D, 0x167282ED, 0x2B12AB5D,
+ 0xA9423C8C, 0x9422153C, 0xD3826FEC, 0xEEE2465C,
+ 0x5CC29A4C, 0x61A2B3FC, 0x2602C92C, 0x1B62E09C,
+ 0xF9D2E0CF, 0xC4B2C97F, 0x8312B3AF, 0xBE729A1F,
+ 0x0C52460F, 0x31326FBF, 0x7692156F, 0x4BF23CDF,
+ 0xC9A2AB0E, 0xF4C282BE, 0xB362F86E, 0x8E02D1DE,
+ 0x3C220DCE, 0x0142247E, 0x46E25EAE, 0x7B82771E,
+ 0xB1E6B092, 0x8C869922, 0xCB26E3F2, 0xF646CA42,
+ 0x44661652, 0x79063FE2, 0x3EA64532, 0x03C66C82,
+ 0x8196FB53, 0xBCF6D2E3, 0xFB56A833, 0xC6368183,
+ 0x74165D93, 0x49767423, 0x0ED60EF3, 0x33B62743,
+ 0xD1062710, 0xEC660EA0, 0xABC67470, 0x96A65DC0,
+ 0x248681D0, 0x19E6A860, 0x5E46D2B0, 0x6326FB00,
+ 0xE1766CD1, 0xDC164561, 0x9BB63FB1, 0xA6D61601,
+ 0x14F6CA11, 0x2996E3A1, 0x6E369971, 0x5356B0C1,
+ 0x70279F96, 0x4D47B626, 0x0AE7CCF6, 0x3787E546,
+ 0x85A73956, 0xB8C710E6, 0xFF676A36, 0xC2074386,
+ 0x4057D457, 0x7D37FDE7, 0x3A978737, 0x07F7AE87,
+ 0xB5D77297, 0x88B75B27, 0xCF1721F7, 0xF2770847,
+ 0x10C70814, 0x2DA721A4, 0x6A075B74, 0x576772C4,
+ 0xE547AED4, 0xD8278764, 0x9F87FDB4, 0xA2E7D404,
+ 0x20B743D5, 0x1DD76A65, 0x5A7710B5, 0x67173905,
+ 0xD537E515, 0xE857CCA5, 0xAFF7B675, 0x92979FC5,
+ 0xE915E8DB, 0xD475C16B, 0x93D5BBBB, 0xAEB5920B,
+ 0x1C954E1B, 0x21F567AB, 0x66551D7B, 0x5B3534CB,
+ 0xD965A31A, 0xE4058AAA, 0xA3A5F07A, 0x9EC5D9CA,
+ 0x2CE505DA, 0x11852C6A, 0x562556BA, 0x6B457F0A,
+ 0x89F57F59, 0xB49556E9, 0xF3352C39, 0xCE550589,
+ 0x7C75D999, 0x4115F029, 0x06B58AF9, 0x3BD5A349,
+ 0xB9853498, 0x84E51D28, 0xC34567F8, 0xFE254E48,
+ 0x4C059258, 0x7165BBE8, 0x36C5C138, 0x0BA5E888,
+ 0x28D4C7DF, 0x15B4EE6F, 0x521494BF, 0x6F74BD0F,
+ 0xDD54611F, 0xE03448AF, 0xA794327F, 0x9AF41BCF,
+ 0x18A48C1E, 0x25C4A5AE, 0x6264DF7E, 0x5F04F6CE,
+ 0xED242ADE, 0xD044036E, 0x97E479BE, 0xAA84500E,
+ 0x4834505D, 0x755479ED, 0x32F4033D, 0x0F942A8D,
+ 0xBDB4F69D, 0x80D4DF2D, 0xC774A5FD, 0xFA148C4D,
+ 0x78441B9C, 0x4524322C, 0x028448FC, 0x3FE4614C,
+ 0x8DC4BD5C, 0xB0A494EC, 0xF704EE3C, 0xCA64C78C,
+ /* T8_5 */
+ 0x00000000, 0xCB5CD3A5, 0x4DC8A10B, 0x869472AE,
+ 0x9B914216, 0x50CD91B3, 0xD659E31D, 0x1D0530B8,
+ 0xEC53826D, 0x270F51C8, 0xA19B2366, 0x6AC7F0C3,
+ 0x77C2C07B, 0xBC9E13DE, 0x3A0A6170, 0xF156B2D5,
+ 0x03D6029B, 0xC88AD13E, 0x4E1EA390, 0x85427035,
+ 0x9847408D, 0x531B9328, 0xD58FE186, 0x1ED33223,
+ 0xEF8580F6, 0x24D95353, 0xA24D21FD, 0x6911F258,
+ 0x7414C2E0, 0xBF481145, 0x39DC63EB, 0xF280B04E,
+ 0x07AC0536, 0xCCF0D693, 0x4A64A43D, 0x81387798,
+ 0x9C3D4720, 0x57619485, 0xD1F5E62B, 0x1AA9358E,
+ 0xEBFF875B, 0x20A354FE, 0xA6372650, 0x6D6BF5F5,
+ 0x706EC54D, 0xBB3216E8, 0x3DA66446, 0xF6FAB7E3,
+ 0x047A07AD, 0xCF26D408, 0x49B2A6A6, 0x82EE7503,
+ 0x9FEB45BB, 0x54B7961E, 0xD223E4B0, 0x197F3715,
+ 0xE82985C0, 0x23755665, 0xA5E124CB, 0x6EBDF76E,
+ 0x73B8C7D6, 0xB8E41473, 0x3E7066DD, 0xF52CB578,
+ 0x0F580A6C, 0xC404D9C9, 0x4290AB67, 0x89CC78C2,
+ 0x94C9487A, 0x5F959BDF, 0xD901E971, 0x125D3AD4,
+ 0xE30B8801, 0x28575BA4, 0xAEC3290A, 0x659FFAAF,
+ 0x789ACA17, 0xB3C619B2, 0x35526B1C, 0xFE0EB8B9,
+ 0x0C8E08F7, 0xC7D2DB52, 0x4146A9FC, 0x8A1A7A59,
+ 0x971F4AE1, 0x5C439944, 0xDAD7EBEA, 0x118B384F,
+ 0xE0DD8A9A, 0x2B81593F, 0xAD152B91, 0x6649F834,
+ 0x7B4CC88C, 0xB0101B29, 0x36846987, 0xFDD8BA22,
+ 0x08F40F5A, 0xC3A8DCFF, 0x453CAE51, 0x8E607DF4,
+ 0x93654D4C, 0x58399EE9, 0xDEADEC47, 0x15F13FE2,
+ 0xE4A78D37, 0x2FFB5E92, 0xA96F2C3C, 0x6233FF99,
+ 0x7F36CF21, 0xB46A1C84, 0x32FE6E2A, 0xF9A2BD8F,
+ 0x0B220DC1, 0xC07EDE64, 0x46EAACCA, 0x8DB67F6F,
+ 0x90B34FD7, 0x5BEF9C72, 0xDD7BEEDC, 0x16273D79,
+ 0xE7718FAC, 0x2C2D5C09, 0xAAB92EA7, 0x61E5FD02,
+ 0x7CE0CDBA, 0xB7BC1E1F, 0x31286CB1, 0xFA74BF14,
+ 0x1EB014D8, 0xD5ECC77D, 0x5378B5D3, 0x98246676,
+ 0x852156CE, 0x4E7D856B, 0xC8E9F7C5, 0x03B52460,
+ 0xF2E396B5, 0x39BF4510, 0xBF2B37BE, 0x7477E41B,
+ 0x6972D4A3, 0xA22E0706, 0x24BA75A8, 0xEFE6A60D,
+ 0x1D661643, 0xD63AC5E6, 0x50AEB748, 0x9BF264ED,
+ 0x86F75455, 0x4DAB87F0, 0xCB3FF55E, 0x006326FB,
+ 0xF135942E, 0x3A69478B, 0xBCFD3525, 0x77A1E680,
+ 0x6AA4D638, 0xA1F8059D, 0x276C7733, 0xEC30A496,
+ 0x191C11EE, 0xD240C24B, 0x54D4B0E5, 0x9F886340,
+ 0x828D53F8, 0x49D1805D, 0xCF45F2F3, 0x04192156,
+ 0xF54F9383, 0x3E134026, 0xB8873288, 0x73DBE12D,
+ 0x6EDED195, 0xA5820230, 0x2316709E, 0xE84AA33B,
+ 0x1ACA1375, 0xD196C0D0, 0x5702B27E, 0x9C5E61DB,
+ 0x815B5163, 0x4A0782C6, 0xCC93F068, 0x07CF23CD,
+ 0xF6999118, 0x3DC542BD, 0xBB513013, 0x700DE3B6,
+ 0x6D08D30E, 0xA65400AB, 0x20C07205, 0xEB9CA1A0,
+ 0x11E81EB4, 0xDAB4CD11, 0x5C20BFBF, 0x977C6C1A,
+ 0x8A795CA2, 0x41258F07, 0xC7B1FDA9, 0x0CED2E0C,
+ 0xFDBB9CD9, 0x36E74F7C, 0xB0733DD2, 0x7B2FEE77,
+ 0x662ADECF, 0xAD760D6A, 0x2BE27FC4, 0xE0BEAC61,
+ 0x123E1C2F, 0xD962CF8A, 0x5FF6BD24, 0x94AA6E81,
+ 0x89AF5E39, 0x42F38D9C, 0xC467FF32, 0x0F3B2C97,
+ 0xFE6D9E42, 0x35314DE7, 0xB3A53F49, 0x78F9ECEC,
+ 0x65FCDC54, 0xAEA00FF1, 0x28347D5F, 0xE368AEFA,
+ 0x16441B82, 0xDD18C827, 0x5B8CBA89, 0x90D0692C,
+ 0x8DD55994, 0x46898A31, 0xC01DF89F, 0x0B412B3A,
+ 0xFA1799EF, 0x314B4A4A, 0xB7DF38E4, 0x7C83EB41,
+ 0x6186DBF9, 0xAADA085C, 0x2C4E7AF2, 0xE712A957,
+ 0x15921919, 0xDECECABC, 0x585AB812, 0x93066BB7,
+ 0x8E035B0F, 0x455F88AA, 0xC3CBFA04, 0x089729A1,
+ 0xF9C19B74, 0x329D48D1, 0xB4093A7F, 0x7F55E9DA,
+ 0x6250D962, 0xA90C0AC7, 0x2F987869, 0xE4C4ABCC,
+ /* T8_6 */
+ 0x00000000, 0xA6770BB4, 0x979F1129, 0x31E81A9D,
+ 0xF44F2413, 0x52382FA7, 0x63D0353A, 0xC5A73E8E,
+ 0x33EF4E67, 0x959845D3, 0xA4705F4E, 0x020754FA,
+ 0xC7A06A74, 0x61D761C0, 0x503F7B5D, 0xF64870E9,
+ 0x67DE9CCE, 0xC1A9977A, 0xF0418DE7, 0x56368653,
+ 0x9391B8DD, 0x35E6B369, 0x040EA9F4, 0xA279A240,
+ 0x5431D2A9, 0xF246D91D, 0xC3AEC380, 0x65D9C834,
+ 0xA07EF6BA, 0x0609FD0E, 0x37E1E793, 0x9196EC27,
+ 0xCFBD399C, 0x69CA3228, 0x582228B5, 0xFE552301,
+ 0x3BF21D8F, 0x9D85163B, 0xAC6D0CA6, 0x0A1A0712,
+ 0xFC5277FB, 0x5A257C4F, 0x6BCD66D2, 0xCDBA6D66,
+ 0x081D53E8, 0xAE6A585C, 0x9F8242C1, 0x39F54975,
+ 0xA863A552, 0x0E14AEE6, 0x3FFCB47B, 0x998BBFCF,
+ 0x5C2C8141, 0xFA5B8AF5, 0xCBB39068, 0x6DC49BDC,
+ 0x9B8CEB35, 0x3DFBE081, 0x0C13FA1C, 0xAA64F1A8,
+ 0x6FC3CF26, 0xC9B4C492, 0xF85CDE0F, 0x5E2BD5BB,
+ 0x440B7579, 0xE27C7ECD, 0xD3946450, 0x75E36FE4,
+ 0xB044516A, 0x16335ADE, 0x27DB4043, 0x81AC4BF7,
+ 0x77E43B1E, 0xD19330AA, 0xE07B2A37, 0x460C2183,
+ 0x83AB1F0D, 0x25DC14B9, 0x14340E24, 0xB2430590,
+ 0x23D5E9B7, 0x85A2E203, 0xB44AF89E, 0x123DF32A,
+ 0xD79ACDA4, 0x71EDC610, 0x4005DC8D, 0xE672D739,
+ 0x103AA7D0, 0xB64DAC64, 0x87A5B6F9, 0x21D2BD4D,
+ 0xE47583C3, 0x42028877, 0x73EA92EA, 0xD59D995E,
+ 0x8BB64CE5, 0x2DC14751, 0x1C295DCC, 0xBA5E5678,
+ 0x7FF968F6, 0xD98E6342, 0xE86679DF, 0x4E11726B,
+ 0xB8590282, 0x1E2E0936, 0x2FC613AB, 0x89B1181F,
+ 0x4C162691, 0xEA612D25, 0xDB8937B8, 0x7DFE3C0C,
+ 0xEC68D02B, 0x4A1FDB9F, 0x7BF7C102, 0xDD80CAB6,
+ 0x1827F438, 0xBE50FF8C, 0x8FB8E511, 0x29CFEEA5,
+ 0xDF879E4C, 0x79F095F8, 0x48188F65, 0xEE6F84D1,
+ 0x2BC8BA5F, 0x8DBFB1EB, 0xBC57AB76, 0x1A20A0C2,
+ 0x8816EAF2, 0x2E61E146, 0x1F89FBDB, 0xB9FEF06F,
+ 0x7C59CEE1, 0xDA2EC555, 0xEBC6DFC8, 0x4DB1D47C,
+ 0xBBF9A495, 0x1D8EAF21, 0x2C66B5BC, 0x8A11BE08,
+ 0x4FB68086, 0xE9C18B32, 0xD82991AF, 0x7E5E9A1B,
+ 0xEFC8763C, 0x49BF7D88, 0x78576715, 0xDE206CA1,
+ 0x1B87522F, 0xBDF0599B, 0x8C184306, 0x2A6F48B2,
+ 0xDC27385B, 0x7A5033EF, 0x4BB82972, 0xEDCF22C6,
+ 0x28681C48, 0x8E1F17FC, 0xBFF70D61, 0x198006D5,
+ 0x47ABD36E, 0xE1DCD8DA, 0xD034C247, 0x7643C9F3,
+ 0xB3E4F77D, 0x1593FCC9, 0x247BE654, 0x820CEDE0,
+ 0x74449D09, 0xD23396BD, 0xE3DB8C20, 0x45AC8794,
+ 0x800BB91A, 0x267CB2AE, 0x1794A833, 0xB1E3A387,
+ 0x20754FA0, 0x86024414, 0xB7EA5E89, 0x119D553D,
+ 0xD43A6BB3, 0x724D6007, 0x43A57A9A, 0xE5D2712E,
+ 0x139A01C7, 0xB5ED0A73, 0x840510EE, 0x22721B5A,
+ 0xE7D525D4, 0x41A22E60, 0x704A34FD, 0xD63D3F49,
+ 0xCC1D9F8B, 0x6A6A943F, 0x5B828EA2, 0xFDF58516,
+ 0x3852BB98, 0x9E25B02C, 0xAFCDAAB1, 0x09BAA105,
+ 0xFFF2D1EC, 0x5985DA58, 0x686DC0C5, 0xCE1ACB71,
+ 0x0BBDF5FF, 0xADCAFE4B, 0x9C22E4D6, 0x3A55EF62,
+ 0xABC30345, 0x0DB408F1, 0x3C5C126C, 0x9A2B19D8,
+ 0x5F8C2756, 0xF9FB2CE2, 0xC813367F, 0x6E643DCB,
+ 0x982C4D22, 0x3E5B4696, 0x0FB35C0B, 0xA9C457BF,
+ 0x6C636931, 0xCA146285, 0xFBFC7818, 0x5D8B73AC,
+ 0x03A0A617, 0xA5D7ADA3, 0x943FB73E, 0x3248BC8A,
+ 0xF7EF8204, 0x519889B0, 0x6070932D, 0xC6079899,
+ 0x304FE870, 0x9638E3C4, 0xA7D0F959, 0x01A7F2ED,
+ 0xC400CC63, 0x6277C7D7, 0x539FDD4A, 0xF5E8D6FE,
+ 0x647E3AD9, 0xC209316D, 0xF3E12BF0, 0x55962044,
+ 0x90311ECA, 0x3646157E, 0x07AE0FE3, 0xA1D90457,
+ 0x579174BE, 0xF1E67F0A, 0xC00E6597, 0x66796E23,
+ 0xA3DE50AD, 0x05A95B19, 0x34414184, 0x92364A30,
+ /* T8_7 */
+ 0x00000000, 0xCCAA009E, 0x4225077D, 0x8E8F07E3,
+ 0x844A0EFA, 0x48E00E64, 0xC66F0987, 0x0AC50919,
+ 0xD3E51BB5, 0x1F4F1B2B, 0x91C01CC8, 0x5D6A1C56,
+ 0x57AF154F, 0x9B0515D1, 0x158A1232, 0xD92012AC,
+ 0x7CBB312B, 0xB01131B5, 0x3E9E3656, 0xF23436C8,
+ 0xF8F13FD1, 0x345B3F4F, 0xBAD438AC, 0x767E3832,
+ 0xAF5E2A9E, 0x63F42A00, 0xED7B2DE3, 0x21D12D7D,
+ 0x2B142464, 0xE7BE24FA, 0x69312319, 0xA59B2387,
+ 0xF9766256, 0x35DC62C8, 0xBB53652B, 0x77F965B5,
+ 0x7D3C6CAC, 0xB1966C32, 0x3F196BD1, 0xF3B36B4F,
+ 0x2A9379E3, 0xE639797D, 0x68B67E9E, 0xA41C7E00,
+ 0xAED97719, 0x62737787, 0xECFC7064, 0x205670FA,
+ 0x85CD537D, 0x496753E3, 0xC7E85400, 0x0B42549E,
+ 0x01875D87, 0xCD2D5D19, 0x43A25AFA, 0x8F085A64,
+ 0x562848C8, 0x9A824856, 0x140D4FB5, 0xD8A74F2B,
+ 0xD2624632, 0x1EC846AC, 0x9047414F, 0x5CED41D1,
+ 0x299DC2ED, 0xE537C273, 0x6BB8C590, 0xA712C50E,
+ 0xADD7CC17, 0x617DCC89, 0xEFF2CB6A, 0x2358CBF4,
+ 0xFA78D958, 0x36D2D9C6, 0xB85DDE25, 0x74F7DEBB,
+ 0x7E32D7A2, 0xB298D73C, 0x3C17D0DF, 0xF0BDD041,
+ 0x5526F3C6, 0x998CF358, 0x1703F4BB, 0xDBA9F425,
+ 0xD16CFD3C, 0x1DC6FDA2, 0x9349FA41, 0x5FE3FADF,
+ 0x86C3E873, 0x4A69E8ED, 0xC4E6EF0E, 0x084CEF90,
+ 0x0289E689, 0xCE23E617, 0x40ACE1F4, 0x8C06E16A,
+ 0xD0EBA0BB, 0x1C41A025, 0x92CEA7C6, 0x5E64A758,
+ 0x54A1AE41, 0x980BAEDF, 0x1684A93C, 0xDA2EA9A2,
+ 0x030EBB0E, 0xCFA4BB90, 0x412BBC73, 0x8D81BCED,
+ 0x8744B5F4, 0x4BEEB56A, 0xC561B289, 0x09CBB217,
+ 0xAC509190, 0x60FA910E, 0xEE7596ED, 0x22DF9673,
+ 0x281A9F6A, 0xE4B09FF4, 0x6A3F9817, 0xA6959889,
+ 0x7FB58A25, 0xB31F8ABB, 0x3D908D58, 0xF13A8DC6,
+ 0xFBFF84DF, 0x37558441, 0xB9DA83A2, 0x7570833C,
+ 0x533B85DA, 0x9F918544, 0x111E82A7, 0xDDB48239,
+ 0xD7718B20, 0x1BDB8BBE, 0x95548C5D, 0x59FE8CC3,
+ 0x80DE9E6F, 0x4C749EF1, 0xC2FB9912, 0x0E51998C,
+ 0x04949095, 0xC83E900B, 0x46B197E8, 0x8A1B9776,
+ 0x2F80B4F1, 0xE32AB46F, 0x6DA5B38C, 0xA10FB312,
+ 0xABCABA0B, 0x6760BA95, 0xE9EFBD76, 0x2545BDE8,
+ 0xFC65AF44, 0x30CFAFDA, 0xBE40A839, 0x72EAA8A7,
+ 0x782FA1BE, 0xB485A120, 0x3A0AA6C3, 0xF6A0A65D,
+ 0xAA4DE78C, 0x66E7E712, 0xE868E0F1, 0x24C2E06F,
+ 0x2E07E976, 0xE2ADE9E8, 0x6C22EE0B, 0xA088EE95,
+ 0x79A8FC39, 0xB502FCA7, 0x3B8DFB44, 0xF727FBDA,
+ 0xFDE2F2C3, 0x3148F25D, 0xBFC7F5BE, 0x736DF520,
+ 0xD6F6D6A7, 0x1A5CD639, 0x94D3D1DA, 0x5879D144,
+ 0x52BCD85D, 0x9E16D8C3, 0x1099DF20, 0xDC33DFBE,
+ 0x0513CD12, 0xC9B9CD8C, 0x4736CA6F, 0x8B9CCAF1,
+ 0x8159C3E8, 0x4DF3C376, 0xC37CC495, 0x0FD6C40B,
+ 0x7AA64737, 0xB60C47A9, 0x3883404A, 0xF42940D4,
+ 0xFEEC49CD, 0x32464953, 0xBCC94EB0, 0x70634E2E,
+ 0xA9435C82, 0x65E95C1C, 0xEB665BFF, 0x27CC5B61,
+ 0x2D095278, 0xE1A352E6, 0x6F2C5505, 0xA386559B,
+ 0x061D761C, 0xCAB77682, 0x44387161, 0x889271FF,
+ 0x825778E6, 0x4EFD7878, 0xC0727F9B, 0x0CD87F05,
+ 0xD5F86DA9, 0x19526D37, 0x97DD6AD4, 0x5B776A4A,
+ 0x51B26353, 0x9D1863CD, 0x1397642E, 0xDF3D64B0,
+ 0x83D02561, 0x4F7A25FF, 0xC1F5221C, 0x0D5F2282,
+ 0x079A2B9B, 0xCB302B05, 0x45BF2CE6, 0x89152C78,
+ 0x50353ED4, 0x9C9F3E4A, 0x121039A9, 0xDEBA3937,
+ 0xD47F302E, 0x18D530B0, 0x965A3753, 0x5AF037CD,
+ 0xFF6B144A, 0x33C114D4, 0xBD4E1337, 0x71E413A9,
+ 0x7B211AB0, 0xB78B1A2E, 0x39041DCD, 0xF5AE1D53,
+ 0x2C8E0FFF, 0xE0240F61, 0x6EAB0882, 0xA201081C,
+ 0xA8C40105, 0x646E019B, 0xEAE10678, 0x264B06E6
+ };
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
new file mode 100644
index 0000000..3fc5637
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/record/HeaderRecord.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.wal.record;
+
+import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
+
+/**
+ * Header record.
+ */
+public class HeaderRecord extends WALRecord {
+ /** */
+ public static final long MAGIC = 0xB0D045A_CE7ED045AL;
+
+ /** Serializer version */
+ private final int ver;
+
+ /**
+ * @param ver Version.
+ */
+ public HeaderRecord(int ver) {
+ this.ver = ver;
+ }
+
+ /**
+ * @return Version.
+ */
+ public int version() {
+ return ver;
+ }
+
+ /** {@inheritDoc} */
+ @Override public RecordType type() {
+ return RecordType.HEADER_RECORD;
+ }
+}
[45/47] ignite git commit: IGNITE-5267 - Moved ignite-ps module to
ignite-core
Posted by ag...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsMXBeanImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsMXBeanImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsMXBeanImpl.java
deleted file mode 100644
index b53db4b..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsMXBeanImpl.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.MemoryMetrics;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.mxbean.MemoryMetricsMXBean;
-
-/**
- * MBean to expose {@link MemoryMetrics} through JMX interface.
- */
-class MemoryMetricsMXBeanImpl implements MemoryMetricsMXBean {
- /** */
- private final MemoryMetricsImpl memMetrics;
-
- /** */
- private final MemoryPolicyConfiguration memPlcCfg;
-
- /**
- * @param memMetrics MemoryMetrics instance to expose through JMX interface.
- * @param memPlcCfg configuration of memory policy this MX Bean is created for.
- */
- MemoryMetricsMXBeanImpl(MemoryMetricsImpl memMetrics,
- MemoryPolicyConfiguration memPlcCfg
- ) {
- this.memMetrics = memMetrics;
- this.memPlcCfg = memPlcCfg;
- }
-
- /** {@inheritDoc} */
- @Override public float getAllocationRate() {
- return memMetrics.getAllocationRate();
- }
-
- /** {@inheritDoc} */
- @Override public float getEvictionRate() {
- return memMetrics.getEvictionRate();
- }
-
- /** {@inheritDoc} */
- @Override public float getLargeEntriesPagesPercentage() {
- return memMetrics.getLargeEntriesPagesPercentage();
- }
-
- /** {@inheritDoc} */
- @Override public float getPagesFillFactor() {
- return memMetrics.getPagesFillFactor();
- }
-
- /** {@inheritDoc} */
- @Override public long getTotalAllocatedPages() {
- return memMetrics.getTotalAllocatedPages();
- }
-
- /** {@inheritDoc} */
- @Override public long getDirtyPages() {
- return memMetrics.getDirtyPages();
- }
-
- /** {@inheritDoc} */
- @Override public float getPagesReplaceRate() {
- return memMetrics.getPagesReplaceRate();
- }
-
- /** {@inheritDoc} */
- @Override public long getPhysicalMemoryPages() {
- return memMetrics.getPhysicalMemoryPages();
- }
-
- /** {@inheritDoc} */
- @Override public void rateTimeInterval(long rateTimeInterval) {
- if (rateTimeInterval < 1000)
- throw new IllegalArgumentException("rateTimeInterval property must be positive " +
- "and greater than 1_000 milliseconds (one second)");
-
- memMetrics.rateTimeInterval(rateTimeInterval);
- }
-
- /** {@inheritDoc} */
- @Override public void subIntervals(int subInts) {
- if (subInts <= 1)
- throw new IllegalArgumentException("subIntervals property must be positive " +
- "and greater than one");
-
- memMetrics.subIntervals(subInts);
- }
-
- /** {@inheritDoc} */
- @Override public void enableMetrics() {
- memMetrics.enableMetrics();
- }
-
- /** {@inheritDoc} */
- @Override public void disableMetrics() {
- memMetrics.disableMetrics();
- }
-
- /** {@inheritDoc} */
- @Override public String getName() {
- return memMetrics.getName();
- }
-
- /** {@inheritDoc} */
- @Override public int getInitialSize() {
- return (int) (memPlcCfg.getInitialSize() / (1024 * 1024));
- }
-
- /** {@inheritDoc} */
- @Override public int getMaxSize() {
- return (int) (memPlcCfg.getMaxSize() / (1024 * 1024));
- }
-
- /** {@inheritDoc} */
- @Override public String getSwapFilePath() {
- return memPlcCfg.getSwapFilePath();
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsSnapshot.java
deleted file mode 100644
index f4874eb..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryMetricsSnapshot.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.MemoryMetrics;
-
-/**
- *
- */
-public class MemoryMetricsSnapshot implements MemoryMetrics {
- /** */
- private String name;
-
- /** */
- private long totalAllocatedPages;
-
- /** */
- private float allocationRate;
-
- /** */
- private float evictionRate;
-
- /** */
- private float largeEntriesPagesPercentage;
-
- /** */
- private float pagesFillFactor;
-
- /** */
- private long dirtyPages;
-
- /** */
- private float pageReplaceRate;
-
- /** */
- private long physicalMemoryPages;
-
- /**
- * @param metrics Metrics instance to take a copy.
- */
- public MemoryMetricsSnapshot(MemoryMetrics metrics) {
- name = metrics.getName();
- totalAllocatedPages = metrics.getTotalAllocatedPages();
- allocationRate = metrics.getAllocationRate();
- evictionRate = metrics.getEvictionRate();
- largeEntriesPagesPercentage = metrics.getLargeEntriesPagesPercentage();
- pagesFillFactor = metrics.getPagesFillFactor();
- dirtyPages = metrics.getDirtyPages();
- pageReplaceRate = metrics.getPagesReplaceRate();
- physicalMemoryPages = metrics.getPhysicalMemoryPages();
- }
-
- /** {@inheritDoc} */
- @Override public String getName() {
- return name;
- }
-
- /** {@inheritDoc} */
- @Override public long getTotalAllocatedPages() {
- return totalAllocatedPages;
- }
-
- /** {@inheritDoc} */
- @Override public float getAllocationRate() {
- return allocationRate;
- }
-
- /** {@inheritDoc} */
- @Override public float getEvictionRate() {
- return evictionRate;
- }
-
- /** {@inheritDoc} */
- @Override public float getLargeEntriesPagesPercentage() {
- return largeEntriesPagesPercentage;
- }
-
- /** {@inheritDoc} */
- @Override public float getPagesFillFactor() {
- return pagesFillFactor;
- }
-
- /** {@inheritDoc} */
- @Override public long getDirtyPages() {
- return dirtyPages;
- }
-
- /** {@inheritDoc} */
- @Override public float getPagesReplaceRate() {
- return pageReplaceRate;
- }
-
- /** {@inheritDoc} */
- @Override public long getPhysicalMemoryPages() {
- return physicalMemoryPages;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java
deleted file mode 100644
index cb35d33..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MemoryPolicy.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.database.evict.PageEvictionTracker;
-
-/**
- * Memory policy provides access to objects configured with {@link MemoryPolicyConfiguration} configuration.
- */
-public class MemoryPolicy {
- /** */
- private final PageMemory pageMem;
-
- /** */
- private final MemoryMetricsImpl memMetrics;
-
- /** */
- private final MemoryPolicyConfiguration cfg;
-
- /** */
- private final PageEvictionTracker evictionTracker;
-
- /**
- * @param pageMem PageMemory instance.
- * @param memMetrics MemoryMetrics instance.
- * @param cfg Configuration of given MemoryPolicy.
- * @param evictionTracker Eviction tracker.
- */
- public MemoryPolicy(
- PageMemory pageMem,
- MemoryPolicyConfiguration cfg,
- MemoryMetricsImpl memMetrics,
- PageEvictionTracker evictionTracker) {
- this.pageMem = pageMem;
- this.memMetrics = memMetrics;
- this.cfg = cfg;
- this.evictionTracker = evictionTracker;
- }
-
- /**
- *
- */
- public PageMemory pageMemory() {
- return pageMem;
- }
-
- /**
- * @return Config.
- */
- public MemoryPolicyConfiguration config() {
- return cfg;
- }
-
- /**
- * @return Memory Metrics.
- */
- public MemoryMetricsImpl memoryMetrics() {
- return memMetrics;
- }
-
- /**
- *
- */
- public PageEvictionTracker evictionTracker() {
- return evictionTracker;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetaStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetaStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetaStore.java
deleted file mode 100644
index c21b818..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetaStore.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.IgniteCheckedException;
-
-/**
- * Meta store.
- */
-public interface MetaStore {
- /**
- * Get or allocate initial page for an index.
- *
- * @param idxName Index name.
- * @return {@link RootPage} that keeps pageId, allocated flag that shows whether the page
- * was newly allocated, and rootId that is counter which increments each time new page allocated.
- * @throws IgniteCheckedException If failed.
- */
- public RootPage getOrAllocateForTree(String idxName) throws IgniteCheckedException;
-
- /**
- * Deallocate index page and remove from tree.
- *
- * @param idxName Index name.
- * @return Root ID or -1 if no page was removed.
- * @throws IgniteCheckedException If failed.
- */
- public RootPage dropRootPage(String idxName) throws IgniteCheckedException;
-
- /**
- * Destroy this meta store.
- *
- * @throws IgniteCheckedException If failed.
- */
- public void destroy() throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
deleted file mode 100644
index 139bf73..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/MetadataStorage.java
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import java.nio.charset.StandardCharsets;
-import java.util.concurrent.atomic.AtomicLong;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.PageUtils;
-import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
-import org.apache.ignite.internal.processors.cache.database.tree.BPlusTree;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusInnerIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.BPlusLeafIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.IOVersions;
-import org.apache.ignite.internal.processors.cache.database.tree.reuse.ReuseList;
-import org.apache.ignite.internal.processors.cache.database.tree.util.PageHandler;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- * Metadata storage.
- */
-public class MetadataStorage implements MetaStore {
- /** Max index name length (bytes num) */
- public static final int MAX_IDX_NAME_LEN = 768;
-
- /** Bytes in byte. */
- private static final int BYTE_LEN = 1;
-
- /** Page memory. */
- private final PageMemory pageMem;
-
- /** Index tree. */
- private final MetaTree metaTree;
-
- /** Meta page reuse tree. */
- private final ReuseList reuseList;
-
- /** Cache group ID. */
- private final int grpId;
-
- /** */
- private final int allocPartId;
-
- /** */
- private final byte allocSpace;
-
- /**
- * @param pageMem Page memory.
- * @param wal Write ahead log manager.
- */
- public MetadataStorage(
- final PageMemory pageMem,
- final IgniteWriteAheadLogManager wal,
- final AtomicLong globalRmvId,
- final int grpId,
- final int allocPartId,
- final byte allocSpace,
- final ReuseList reuseList,
- final long rootPageId,
- final boolean initNew
- ) {
- try {
- this.pageMem = pageMem;
- this.grpId = grpId;
- this.allocPartId = allocPartId;
- this.allocSpace = allocSpace;
- this.reuseList = reuseList;
-
- metaTree = new MetaTree(grpId, allocPartId, allocSpace, pageMem, wal, globalRmvId, rootPageId,
- reuseList, MetaStoreInnerIO.VERSIONS, MetaStoreLeafIO.VERSIONS, initNew);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- }
-
- /** {@inheritDoc} */
- @Override public RootPage getOrAllocateForTree(final String idxName) throws IgniteCheckedException {
- final MetaTree tree = metaTree;
-
- synchronized (this) {
- byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
-
- if (idxNameBytes.length > MAX_IDX_NAME_LEN)
- throw new IllegalArgumentException("Too long encoded indexName [maxAllowed=" + MAX_IDX_NAME_LEN +
- ", currentLength=" + idxNameBytes.length + ", name=" + idxName + "]");
-
- final IndexItem row = tree.findOne(new IndexItem(idxNameBytes, 0));
-
- if (row == null) {
- long pageId = 0;
-
- if (reuseList != null)
- pageId = reuseList.takeRecycledPage();
-
- pageId = pageId == 0 ? pageMem.allocatePage(grpId, allocPartId, allocSpace) : pageId;
-
- tree.put(new IndexItem(idxNameBytes, pageId));
-
- return new RootPage(new FullPageId(pageId, grpId), true);
- }
- else {
- final FullPageId pageId = new FullPageId(row.pageId, grpId);
-
- return new RootPage(pageId, false);
- }
- }
- }
-
- /** {@inheritDoc} */
- @Override public RootPage dropRootPage(final String idxName)
- throws IgniteCheckedException {
- byte[] idxNameBytes = idxName.getBytes(StandardCharsets.UTF_8);
-
- final IndexItem row = metaTree.remove(new IndexItem(idxNameBytes, 0));
-
- if (row != null) {
- if (reuseList == null)
- pageMem.freePage(grpId, row.pageId);
- }
-
- return row != null ? new RootPage(new FullPageId(row.pageId, grpId), false) : null;
- }
-
- /** {@inheritDoc} */
- @Override public void destroy() throws IgniteCheckedException {
- metaTree.destroy();
- }
-
- /**
- *
- */
- private static class MetaTree extends BPlusTree<IndexItem, IndexItem> {
- /** */
- private final int allocPartId;
-
- /** */
- private final byte allocSpace;
-
- /**
- * @param pageMem Page memory.
- * @param metaPageId Meta page ID.
- * @param reuseList Reuse list.
- * @param innerIos Inner IOs.
- * @param leafIos Leaf IOs.
- * @throws IgniteCheckedException If failed.
- */
- private MetaTree(
- final int cacheId,
- final int allocPartId,
- final byte allocSpace,
- final PageMemory pageMem,
- final IgniteWriteAheadLogManager wal,
- final AtomicLong globalRmvId,
- final long metaPageId,
- final ReuseList reuseList,
- final IOVersions<? extends BPlusInnerIO<IndexItem>> innerIos,
- final IOVersions<? extends BPlusLeafIO<IndexItem>> leafIos,
- final boolean initNew
- ) throws IgniteCheckedException {
- super(treeName("meta", "Meta"), cacheId, pageMem, wal, globalRmvId, metaPageId, reuseList, innerIos, leafIos);
-
- this.allocPartId = allocPartId;
- this.allocSpace = allocSpace;
-
- initTree(initNew);
- }
-
- /** {@inheritDoc} */
- @Override protected long allocatePageNoReuse() throws IgniteCheckedException {
- return pageMem.allocatePage(getCacheId(), allocPartId, allocSpace);
- }
-
- /** {@inheritDoc} */
- @Override protected int compare(final BPlusIO<IndexItem> io, final long pageAddr, final int idx,
- final IndexItem row) throws IgniteCheckedException {
- final int off = ((IndexIO)io).getOffset(pageAddr, idx);
-
- int shift = 0;
-
- // Compare index names.
- final byte len = PageUtils.getByte(pageAddr, off + shift);
-
- shift += BYTE_LEN;
-
- for (int i = 0; i < len && i < row.idxName.length; i++) {
- final int cmp = Byte.compare(PageUtils.getByte(pageAddr, off + i + shift), row.idxName[i]);
-
- if (cmp != 0)
- return cmp;
- }
-
- return Integer.compare(len, row.idxName.length);
- }
-
- /** {@inheritDoc} */
- @Override protected IndexItem getRow(final BPlusIO<IndexItem> io, final long pageAddr,
- final int idx, Object ignore) throws IgniteCheckedException {
- return readRow(pageAddr, ((IndexIO)io).getOffset(pageAddr, idx));
- }
- }
-
- /**
- *
- */
- private static class IndexItem {
- /** */
- private byte[] idxName;
-
- /** */
- private long pageId;
-
- /**
- * @param idxName Index name.
- * @param pageId Page ID.
- */
- private IndexItem(final byte[] idxName, final long pageId) {
- this.idxName = idxName;
- this.pageId = pageId;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return "I [idxName=" + new String(idxName) + ", pageId=" + U.hexLong(pageId) + ']';
- }
- }
-
- /**
- * Store row to buffer.
- *
- * @param pageAddr Page address.
- * @param off Offset in buf.
- * @param row Row to store.
- */
- private static void storeRow(
- final long pageAddr,
- int off,
- final IndexItem row
- ) {
- // Index name length.
- PageUtils.putByte(pageAddr, off, (byte)row.idxName.length);
- off++;
-
- // Index name.
- PageUtils.putBytes(pageAddr, off, row.idxName);
- off += row.idxName.length;
-
- // Page ID.
- PageUtils.putLong(pageAddr, off, row.pageId);
- }
-
- /**
- * Copy row data.
- *
- * @param dstPageAddr Destination page address.
- * @param dstOff Destination buf offset.
- * @param srcPageAddr Source page address.
- * @param srcOff Src buf offset.
- */
- private static void storeRow(
- final long dstPageAddr,
- int dstOff,
- final long srcPageAddr,
- int srcOff
- ) {
- // Index name length.
- final byte len = PageUtils.getByte(srcPageAddr, srcOff);
- srcOff++;
-
- PageUtils.putByte(dstPageAddr, dstOff, len);
- dstOff++;
-
- PageHandler.copyMemory(srcPageAddr, dstPageAddr, srcOff, dstOff, len);
- srcOff += len;
- dstOff += len;
-
- // Page ID.
- PageUtils.putLong(dstPageAddr, dstOff, PageUtils.getLong(srcPageAddr, srcOff));
- }
-
- /**
- * Read row from buffer.
- *
- * @param pageAddr Page address.
- * @param off Offset.
- * @return Read row.
- */
- private static IndexItem readRow(final long pageAddr, int off) {
- // Index name length.
- final int len = PageUtils.getByte(pageAddr, off) & 0xFF;
- off++;
-
- // Index name.
- final byte[] idxName = PageUtils.getBytes(pageAddr, off, len);
- off += len;
-
- // Page ID.
- final long pageId = PageUtils.getLong(pageAddr, off);
-
- return new IndexItem(idxName, pageId);
- }
-
- /**
- *
- */
- private interface IndexIO {
- /**
- * @param pageAddr Page address.
- * @param idx Index.
- * @return Offset in buffer according to {@code idx}.
- */
- int getOffset(long pageAddr, int idx);
- }
-
- /**
- *
- */
- public static final class MetaStoreInnerIO extends BPlusInnerIO<IndexItem> implements IndexIO {
- /** */
- public static final IOVersions<MetaStoreInnerIO> VERSIONS = new IOVersions<>(
- new MetaStoreInnerIO(1)
- );
-
- /**
- * @param ver Version.
- */
- private MetaStoreInnerIO(final int ver) {
- // name bytes and 1 byte for length, 8 bytes pageId
- super(T_METASTORE_INNER, ver, false, MAX_IDX_NAME_LEN + 1 + 8);
- }
-
- /** {@inheritDoc} */
- @Override public void storeByOffset(long pageAddr, int off, IndexItem row) throws IgniteCheckedException {
- storeRow(pageAddr, off, row);
- }
-
- /** {@inheritDoc} */
- @Override public void store(final long dstPageAddr, final int dstIdx, final BPlusIO<IndexItem> srcIo,
- final long srcPageAddr,
- final int srcIdx) throws IgniteCheckedException {
- storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
- }
-
- /** {@inheritDoc} */
- @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree, final long pageAddr,
- final int idx) throws IgniteCheckedException {
- return readRow(pageAddr, offset(idx));
- }
-
- /** {@inheritDoc} */
- @Override public int getOffset(long pageAddr, final int idx) {
- return offset(idx);
- }
- }
-
- /**
- *
- */
- public static final class MetaStoreLeafIO extends BPlusLeafIO<IndexItem> implements IndexIO {
- /** */
- public static final IOVersions<MetaStoreLeafIO> VERSIONS = new IOVersions<>(
- new MetaStoreLeafIO(1)
- );
-
- /**
- * @param ver Version.
- */
- private MetaStoreLeafIO(final int ver) {
- // 4 byte cache ID, UTF-16 symbols and 1 byte for length, 8 bytes pageId
- super(T_METASTORE_LEAF, ver, MAX_IDX_NAME_LEN + 1 + 8);
- }
-
- /** {@inheritDoc} */
- @Override public void storeByOffset(long buf, int off, IndexItem row) throws IgniteCheckedException {
- storeRow(buf, off, row);
- }
-
- /** {@inheritDoc} */
- @Override public void store(final long dstPageAddr,
- final int dstIdx,
- final BPlusIO<IndexItem> srcIo,
- final long srcPageAddr,
- final int srcIdx) throws IgniteCheckedException {
- storeRow(dstPageAddr, offset(dstIdx), srcPageAddr, ((IndexIO)srcIo).getOffset(srcPageAddr, srcIdx));
- }
-
- /** {@inheritDoc} */
- @Override public IndexItem getLookupRow(final BPlusTree<IndexItem, ?> tree,
- final long pageAddr,
- final int idx) throws IgniteCheckedException {
- return readRow(pageAddr, offset(idx));
- }
-
- /** {@inheritDoc} */
- @Override public int getOffset(long pageAddr, final int idx) {
- return offset(idx);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsSnapshot.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsSnapshot.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsSnapshot.java
deleted file mode 100644
index c939710..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/PersistenceMetricsSnapshot.java
+++ /dev/null
@@ -1,144 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.PersistenceMetrics;
-import org.apache.ignite.internal.util.typedef.internal.S;
-
-/**
- *
- */
-public class PersistenceMetricsSnapshot implements PersistenceMetrics {
- /** */
- private float walLoggingRate;
-
- /** */
- private float walWritingRate;
-
- /** */
- private int walArchiveSegments;
-
- /** */
- private float walFsyncTimeAvg;
-
- /** */
- private long lastCpDuration;
-
- /** */
- private long lastCpLockWaitDuration;
-
- /** */
- private long lastCpMmarkDuration;
-
- /** */
- private long lastCpPagesWriteDuration;
-
- /** */
- private long lastCpFsyncDuration;
-
- /** */
- private long lastCpTotalPages;
-
- /** */
- private long lastCpDataPages;
-
- /** */
- private long lastCpCowPages;
-
- /**
- * @param metrics Metrics.
- */
- public PersistenceMetricsSnapshot(PersistenceMetrics metrics) {
- walLoggingRate = metrics.getWalLoggingRate();
- walWritingRate = metrics.getWalWritingRate();
- walArchiveSegments = metrics.getWalArchiveSegments();
- walFsyncTimeAvg = metrics.getWalFsyncTimeAverage();
- lastCpDuration = metrics.getLastCheckpointingDuration();
- lastCpLockWaitDuration = metrics.getLastCheckpointLockWaitDuration();
- lastCpMmarkDuration = metrics.getLastCheckpointMarkDuration();
- lastCpPagesWriteDuration = metrics.getLastCheckpointPagesWriteDuration();
- lastCpFsyncDuration = metrics.getLastCheckpointFsyncDuration();
- lastCpTotalPages = metrics.getLastCheckpointTotalPagesNumber();
- lastCpDataPages = metrics.getLastCheckpointDataPagesNumber();
- lastCpCowPages = metrics.getLastCheckpointCopiedOnWritePagesNumber();
- }
-
- /** {@inheritDoc} */
- @Override public float getWalLoggingRate() {
- return walLoggingRate;
- }
-
- /** {@inheritDoc} */
- @Override public float getWalWritingRate() {
- return walWritingRate;
- }
-
- /** {@inheritDoc} */
- @Override public int getWalArchiveSegments() {
- return walArchiveSegments;
- }
-
- /** {@inheritDoc} */
- @Override public float getWalFsyncTimeAverage() {
- return walFsyncTimeAvg;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointingDuration() {
- return lastCpDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointLockWaitDuration() {
- return lastCpLockWaitDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointMarkDuration() {
- return lastCpMmarkDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointPagesWriteDuration() {
- return lastCpPagesWriteDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointFsyncDuration() {
- return lastCpFsyncDuration;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointTotalPagesNumber() {
- return lastCpTotalPages;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointDataPagesNumber() {
- return lastCpDataPages;
- }
-
- /** {@inheritDoc} */
- @Override public long getLastCheckpointCopiedOnWritePagesNumber() {
- return lastCpCowPages;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(PersistenceMetricsSnapshot.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RootPage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RootPage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RootPage.java
deleted file mode 100644
index 935046d..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RootPage.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.internal.pagemem.FullPageId;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-
-/**
- *
- */
-public class RootPage {
- /** */
- @GridToStringInclude
- private FullPageId pageId;
-
- /** */
- private boolean allocated;
-
- /**
- * @param pageId Page ID.
- * @param allocated {@code True} if was allocated new page.
- */
- public RootPage(final FullPageId pageId, final boolean allocated) {
- this.pageId = pageId;
- this.allocated = allocated;
- }
-
- /**
- * @return Page ID.
- */
- public FullPageId pageId() {
- return pageId;
- }
-
- /**
- * @return {@code True} if was allocated.
- */
- public boolean isAllocated() {
- return allocated;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(RootPage.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
deleted file mode 100644
index d707869..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/RowStore.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.processors.cache.CacheGroupContext;
-import org.apache.ignite.internal.processors.cache.CacheObjectContext;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.freelist.FreeList;
-
-/**
- * Data store for H2 rows.
- */
-public class RowStore {
- /** */
- private final FreeList freeList;
-
- /** */
- private final GridCacheSharedContext ctx;
-
- /** */
- protected final PageMemory pageMem;
-
- /** */
- protected final CacheObjectContext coctx;
-
-
-
- /**
- * @param grp Cache group.
- * @param freeList Free list.
- */
- public RowStore(CacheGroupContext grp, FreeList freeList) {
- assert grp != null;
- assert freeList != null;
-
- this.freeList = freeList;
-
- ctx = grp.shared();
- coctx = grp.cacheObjectContext();
- pageMem = grp.memoryPolicy().pageMemory();
- }
-
- /**
- * @param link Row link.
- * @throws IgniteCheckedException If failed.
- */
- public void removeRow(long link) throws IgniteCheckedException {
- assert link != 0;
- ctx.database().checkpointReadLock();
-
- try {
- freeList.removeDataRowByLink(link);
- }
- finally {
- ctx.database().checkpointReadUnlock();
- }
- }
-
- /**
- * @param row Row.
- * @throws IgniteCheckedException If failed.
- */
- public void addRow(CacheDataRow row) throws IgniteCheckedException {
- ctx.database().checkpointReadLock();
-
- try {
- freeList.insertDataRow(row);
- }
- finally {
- ctx.database().checkpointReadUnlock();
- }
- }
-
- /**
- * @param link Row link.
- * @param row New row data.
- * @throws IgniteCheckedException If failed.
- * @return {@code True} if was able to update row.
- */
- public boolean updateRow(long link, CacheDataRow row) throws IgniteCheckedException {
- return freeList.updateDataRow(link, row);
- }
-
- /**
- * @return Free list.
- */
- public FreeList freeList() {
- return freeList;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
deleted file mode 100644
index b7c6b57..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/FairFifoPageEvictionTracker.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.ignite.internal.processors.cache.database.evict;
-
-import java.util.LinkedList;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-
-/**
- * On-heap FIFO page eviction tracker. Only for test purposes.
- */
-public class FairFifoPageEvictionTracker extends PageAbstractEvictionTracker {
- /** Page usage deque. */
- private final LinkedList<Integer> pageUsageList = new LinkedList<>();
-
- /**
- * @param pageMem Page memory.
- * @param plcCfg Memory policy configuration.
- * @param sharedCtx Shared context.
- */
- public FairFifoPageEvictionTracker(
- PageMemoryNoStoreImpl pageMem,
- MemoryPolicyConfiguration plcCfg,
- GridCacheSharedContext sharedCtx) {
- super(pageMem, plcCfg, sharedCtx);
- }
-
- /** {@inheritDoc} */
- @Override public void start() throws IgniteException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void stop() throws IgniteException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void touchPage(long pageId) throws IgniteCheckedException {
- pageUsageList.addLast(PageIdUtils.pageIndex(pageId));
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void evictDataPage() throws IgniteCheckedException {
- evictDataPage(pageUsageList.pollFirst());
- }
-
- /** {@inheritDoc} */
- @Override public synchronized void forgetPage(long pageId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override protected synchronized boolean checkTouch(long pageId) {
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/NoOpPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/NoOpPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/NoOpPageEvictionTracker.java
deleted file mode 100644
index ba466bf..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/NoOpPageEvictionTracker.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.ignite.internal.processors.cache.database.evict;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-
-/**
- *
- */
-public class NoOpPageEvictionTracker implements PageEvictionTracker {
- /** {@inheritDoc} */
- @Override public void start() throws IgniteException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void stop() throws IgniteException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void touchPage(long pageId) throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void evictDataPage() throws IgniteCheckedException {
- // No-op.
- }
-
- /** {@inheritDoc} */
- @Override public void forgetPage(long pageId) throws IgniteCheckedException {
- // No-op.
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
deleted file mode 100644
index c61aced..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageAbstractEvictionTracker.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.ignite.internal.processors.cache.database.evict;
-
-import java.util.List;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRowAdapter;
-import org.apache.ignite.internal.processors.cache.database.tree.io.DataPageIO;
-import org.apache.ignite.internal.processors.cache.database.tree.io.PageIO;
-import org.apache.ignite.internal.processors.cache.version.GridCacheVersionManager;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public abstract class PageAbstractEvictionTracker implements PageEvictionTracker {
- /** This number of least significant bits is dropped from timestamp. */
- private static final int COMPACT_TS_SHIFT = 8; // Enough if grid works for less than 17 years.
-
- /** Millis in day. */
- private static final int DAY = 24 * 60 * 60 * 1000;
-
- /** Page memory. */
- protected final PageMemoryNoStoreImpl pageMem;
-
- /** Tracking array size. */
- protected final int trackingSize;
-
- /** Base compact timestamp. */
- private final long baseCompactTs;
-
- /** Shared context. */
- private final GridCacheSharedContext sharedCtx;
-
- /**
- * @param pageMem Page memory.
- * @param plcCfg Memory policy configuration.
- * @param sharedCtx Shared context.
- */
- PageAbstractEvictionTracker(
- PageMemoryNoStoreImpl pageMem,
- MemoryPolicyConfiguration plcCfg,
- GridCacheSharedContext sharedCtx
- ) {
- this.pageMem = pageMem;
-
- this.sharedCtx = sharedCtx;
-
- trackingSize = pageMem.totalPages();
-
- baseCompactTs = (U.currentTimeMillis() - DAY) >> COMPACT_TS_SHIFT;
- // We subtract day to avoid fail in case of daylight shift or timezone change.
- }
-
- /**
- * @param pageIdx Page index.
- * @return true if at least one data row has been evicted
- * @throws IgniteCheckedException If failed.
- */
- final boolean evictDataPage(int pageIdx) throws IgniteCheckedException {
- long fakePageId = PageIdUtils.pageId(0, (byte)0, pageIdx);
-
- long page = pageMem.acquirePage(0, fakePageId);
-
- List<CacheDataRowAdapter> rowsToEvict;
-
- try {
- long pageAddr = pageMem.readLockForce(0, fakePageId, page);
-
- try {
- if (PageIO.getType(pageAddr) != PageIO.T_DATA)
- return false; // Can't evict: page has been recycled into non-data page.
-
- DataPageIO io = DataPageIO.VERSIONS.forPage(pageAddr);
-
- long realPageId = PageIO.getPageId(pageAddr);
-
- if (!checkTouch(realPageId))
- return false; // Can't evict: another thread concurrently invoked forgetPage()
-
- rowsToEvict = io.forAllItems(pageAddr, new DataPageIO.CC<CacheDataRowAdapter>() {
- @Override public CacheDataRowAdapter apply(long link) throws IgniteCheckedException {
- CacheDataRowAdapter row = new CacheDataRowAdapter(link);
-
- row.initFromLink(null, sharedCtx, pageMem, CacheDataRowAdapter.RowData.KEY_ONLY);
-
- assert row.cacheId() != 0 : "Cache ID should be stored in rows of evictable cache";
-
- return row;
- }
- });
- }
- finally {
- pageMem.readUnlock(0, fakePageId, page);
- }
- }
- finally {
- pageMem.releasePage(0, fakePageId, page);
- }
-
- boolean evictionDone = false;
-
- for (CacheDataRowAdapter dataRow : rowsToEvict) {
- GridCacheContext<?, ?> cacheCtx = sharedCtx.cacheContext(dataRow.cacheId());
-
- if (!cacheCtx.userCache())
- continue;
-
- GridCacheEntryEx entryEx = cacheCtx.isNear() ? cacheCtx.near().dht().entryEx(dataRow.key()) :
- cacheCtx.cache().entryEx(dataRow.key());
-
- evictionDone |= entryEx.evictInternal(GridCacheVersionManager.EVICT_VER, null, true);
- }
-
- return evictionDone;
- }
-
- /**
- * @param pageId Page ID.
- * @return true if page was touched at least once.
- */
- protected abstract boolean checkTouch(long pageId);
-
- /**
- * @param epochMilli Time millis.
- * @return Compact timestamp. Comparable and fits in 4 bytes.
- */
- final long compactTimestamp(long epochMilli) {
- return (epochMilli >> COMPACT_TS_SHIFT) - baseCompactTs;
- }
-
- /**
- * Resolves position in tracking array by page index.
- *
- * @param pageIdx Page index.
- * @return Position of page in tracking array.
- */
- int trackingIdx(int pageIdx) {
- return pageMem.pageSequenceNumber(pageIdx);
- }
-
- /**
- * Reverse of {@link #trackingIdx(int)}.
- *
- * @param trackingIdx Tracking index.
- * @return Page index.
- */
- int pageIdx(int trackingIdx) {
- return pageMem.pageIndex(trackingIdx);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageEvictionTracker.java
deleted file mode 100644
index b13dcf8..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/PageEvictionTracker.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.ignite.internal.processors.cache.database.evict;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.lifecycle.LifecycleAware;
-
-/**
- * Entry point for per-page eviction. Accepts information about touching data pages,
- * capable of evicting "the least needed" page (according to implemented eviction algorithm).
- */
-public interface PageEvictionTracker extends LifecycleAware {
- /**
- * Call this method when data page is accessed.
- *
- * @param pageId Page id.
- * @throws IgniteCheckedException In case of page memory error.
- */
- public void touchPage(long pageId) throws IgniteCheckedException;
-
- /**
- * Evicts one data page.
- * In most cases, all entries will be removed from the page.
- * Method guarantees removing at least one entry from "evicted" data page. Removing all entries may be
- * not possible, as some of them can be used by active transactions.
- *
- * @throws IgniteCheckedException In case of page memory error.
- */
- public void evictDataPage() throws IgniteCheckedException;
-
- /**
- * Call this method when last entry is removed from data page.
- *
- * @param pageId Page id.
- * @throws IgniteCheckedException In case of page memory error.
- */
- public void forgetPage(long pageId) throws IgniteCheckedException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
deleted file mode 100644
index 21ebba1..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/Random2LruPageEvictionTracker.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.ignite.internal.processors.cache.database.evict;
-
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.typedef.internal.LT;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class Random2LruPageEvictionTracker extends PageAbstractEvictionTracker {
- /** Evict attempts limit. */
- private static final int EVICT_ATTEMPTS_LIMIT = 30;
-
- /** LRU Sample size. */
- private static final int SAMPLE_SIZE = 5;
-
- /** Maximum sample search spin count */
- private static final int SAMPLE_SPIN_LIMIT = SAMPLE_SIZE * 1000;
-
- /** Logger. */
- private final IgniteLogger log;
-
- /** Tracking array ptr. */
- private long trackingArrPtr;
-
- /**
- * @param pageMem Page memory.
- * @param plcCfg Policy config.
- * @param sharedCtx Shared context.
- */
- public Random2LruPageEvictionTracker(
- PageMemoryNoStoreImpl pageMem,
- MemoryPolicyConfiguration plcCfg,
- GridCacheSharedContext<?, ?> sharedCtx
- ) {
- super(pageMem, plcCfg, sharedCtx);
-
- MemoryConfiguration memCfg = sharedCtx.kernalContext().config().getMemoryConfiguration();
-
- assert plcCfg.getMaxSize() / memCfg.getPageSize() < Integer.MAX_VALUE;
-
- log = sharedCtx.logger(getClass());
- }
-
- /** {@inheritDoc} */
- @Override public void start() throws IgniteException {
- trackingArrPtr = GridUnsafe.allocateMemory(trackingSize * 8);
-
- GridUnsafe.setMemory(trackingArrPtr, trackingSize * 8, (byte)0);
- }
-
- /** {@inheritDoc} */
- @Override public void stop() throws IgniteException {
- GridUnsafe.freeMemory(trackingArrPtr);
- }
-
- /** {@inheritDoc} */
- @Override public void touchPage(long pageId) throws IgniteCheckedException {
- int pageIdx = PageIdUtils.pageIndex(pageId);
-
- long latestTs = compactTimestamp(U.currentTimeMillis());
-
- assert latestTs >= 0 && latestTs < Integer.MAX_VALUE;
-
- boolean success;
-
- do {
- int trackingIdx = trackingIdx(pageIdx);
-
- int firstTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8);
-
- int secondTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8 + 4);
-
- if (firstTs <= secondTs)
- success = GridUnsafe.compareAndSwapInt(null, trackingArrPtr + trackingIdx * 8, firstTs, (int)latestTs);
- else {
- success = GridUnsafe.compareAndSwapInt(
- null, trackingArrPtr + trackingIdx * 8 + 4, secondTs, (int)latestTs);
- }
- } while (!success);
- }
-
- /** {@inheritDoc} */
- @Override public void evictDataPage() throws IgniteCheckedException {
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- int evictAttemptsCnt = 0;
-
- while (evictAttemptsCnt < EVICT_ATTEMPTS_LIMIT) {
- int lruTrackingIdx = -1;
-
- int lruCompactTs = Integer.MAX_VALUE;
-
- int dataPagesCnt = 0;
-
- int sampleSpinCnt = 0;
-
- while (dataPagesCnt < SAMPLE_SIZE) {
- int trackingIdx = rnd.nextInt(trackingSize);
-
- int firstTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8);
-
- int secondTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8 + 4);
-
- int minTs = Math.min(firstTs, secondTs);
-
- int maxTs = Math.max(firstTs, secondTs);
-
- if (maxTs != 0) {
- // We chose data page with at least one touch.
- if (minTs < lruCompactTs) {
- lruTrackingIdx = trackingIdx;
-
- lruCompactTs = minTs;
- }
-
- dataPagesCnt++;
- }
-
- sampleSpinCnt++;
-
- if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) {
- LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT);
-
- return;
- }
- }
-
- if (evictDataPage(pageIdx(lruTrackingIdx)))
- return;
-
- evictAttemptsCnt++;
- }
-
- LT.warn(log, "Too many failed attempts to evict page: " + EVICT_ATTEMPTS_LIMIT);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean checkTouch(long pageId) {
- int trackingIdx = trackingIdx(PageIdUtils.pageIndex(pageId));
-
- int firstTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 8);
-
- return firstTs != 0;
- }
-
- /** {@inheritDoc} */
- @Override public void forgetPage(long pageId) {
- int pageIdx = PageIdUtils.pageIndex(pageId);
-
- int trackingIdx = trackingIdx(pageIdx);
-
- GridUnsafe.putLongVolatile(null, trackingArrPtr + trackingIdx * 8, 0L);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
deleted file mode 100644
index d241148..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/evict/RandomLruPageEvictionTracker.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
-* Licensed to the Apache Software Foundation (ASF) under one or more
-* contributor license agreements. See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The ASF licenses this file to You under the Apache License, Version 2.0
-* (the "License"); you may not use this file except in compliance with
-* the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.ignite.internal.processors.cache.database.evict;
-
-import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.configuration.MemoryConfiguration;
-import org.apache.ignite.configuration.MemoryPolicyConfiguration;
-import org.apache.ignite.internal.pagemem.PageIdUtils;
-import org.apache.ignite.internal.pagemem.PageMemory;
-import org.apache.ignite.internal.pagemem.impl.PageMemoryNoStoreImpl;
-import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
-import org.apache.ignite.internal.util.GridUnsafe;
-import org.apache.ignite.internal.util.typedef.internal.LT;
-import org.apache.ignite.internal.util.typedef.internal.U;
-
-/**
- *
- */
-public class RandomLruPageEvictionTracker extends PageAbstractEvictionTracker {
- /** Evict attempts limit. */
- private static final int EVICT_ATTEMPTS_LIMIT = 30;
-
- /** LRU Sample size. */
- private static final int SAMPLE_SIZE = 5;
-
- /** Maximum sample search spin count */
- private static final int SAMPLE_SPIN_LIMIT = SAMPLE_SIZE * 1000;
-
- /** Logger. */
- private final IgniteLogger log;
-
- /** Tracking array ptr. */
- private long trackingArrPtr;
-
- /**
- * @param pageMem Page memory.
- * @param plcCfg Policy config.
- * @param sharedCtx Shared context.
- */
- public RandomLruPageEvictionTracker(
- PageMemory pageMem,
- MemoryPolicyConfiguration plcCfg,
- GridCacheSharedContext<?, ?> sharedCtx
- ) {
- super((PageMemoryNoStoreImpl)pageMem, plcCfg, sharedCtx);
-
- MemoryConfiguration memCfg = sharedCtx.kernalContext().config().getMemoryConfiguration();
-
- assert plcCfg.getMaxSize() / memCfg.getPageSize() < Integer.MAX_VALUE;
-
- log = sharedCtx.logger(getClass());
- }
-
- /** {@inheritDoc} */
- @Override public void start() throws IgniteException {
- trackingArrPtr = GridUnsafe.allocateMemory(trackingSize * 4);
-
- GridUnsafe.setMemory(trackingArrPtr, trackingSize * 4, (byte)0);
- }
-
- /** {@inheritDoc} */
- @Override public void stop() throws IgniteException {
- GridUnsafe.freeMemory(trackingArrPtr);
- }
-
- /** {@inheritDoc} */
- @Override public void touchPage(long pageId) throws IgniteCheckedException {
- int pageIdx = PageIdUtils.pageIndex(pageId);
-
- long res = compactTimestamp(U.currentTimeMillis());
-
- assert res >= 0 && res < Integer.MAX_VALUE;
-
- GridUnsafe.putIntVolatile(null, trackingArrPtr + trackingIdx(pageIdx) * 4, (int)res);
- }
-
- /** {@inheritDoc} */
- @Override public void evictDataPage() throws IgniteCheckedException {
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- int evictAttemptsCnt = 0;
-
- while (evictAttemptsCnt < EVICT_ATTEMPTS_LIMIT) {
- int lruTrackingIdx = -1;
-
- int lruCompactTs = Integer.MAX_VALUE;
-
- int dataPagesCnt = 0;
-
- int sampleSpinCnt = 0;
-
- while (dataPagesCnt < SAMPLE_SIZE) {
- int sampleTrackingIdx = rnd.nextInt(trackingSize);
-
- int compactTs = GridUnsafe.getIntVolatile(null, trackingArrPtr + sampleTrackingIdx * 4);
-
- if (compactTs != 0) {
- // We chose data page with at least one touch.
- if (compactTs < lruCompactTs) {
- lruTrackingIdx = sampleTrackingIdx;
-
- lruCompactTs = compactTs;
- }
-
- dataPagesCnt++;
- }
-
- sampleSpinCnt++;
-
- if (sampleSpinCnt > SAMPLE_SPIN_LIMIT) {
- LT.warn(log, "Too many attempts to choose data page: " + SAMPLE_SPIN_LIMIT);
-
- return;
- }
- }
-
- if (evictDataPage(pageIdx(lruTrackingIdx)))
- return;
-
- evictAttemptsCnt++;
- }
-
- LT.warn(log, "Too many failed attempts to evict page: " + EVICT_ATTEMPTS_LIMIT);
- }
-
- /** {@inheritDoc} */
- @Override protected boolean checkTouch(long pageId) {
- int trackingIdx = trackingIdx(PageIdUtils.pageIndex(pageId));
-
- int ts = GridUnsafe.getIntVolatile(null, trackingArrPtr + trackingIdx * 4);
-
- return ts != 0;
- }
-
- /** {@inheritDoc} */
- @Override public void forgetPage(long pageId) {
- int pageIdx = PageIdUtils.pageIndex(pageId);
-
- GridUnsafe.putIntVolatile(null, trackingArrPtr + trackingIdx(pageIdx) * 4, 0);
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/6bf5ce46/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
deleted file mode 100644
index 3266b95..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/database/freelist/FreeList.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.cache.database.freelist;
-
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.processors.cache.database.CacheDataRow;
-
-/**
- */
-public interface FreeList {
- /**
- * @param row Row.
- * @throws IgniteCheckedException If failed.
- */
- public void insertDataRow(CacheDataRow row) throws IgniteCheckedException;
-
- /**
- * @param link Row link.
- * @param row New row data.
- * @return {@code True} if was able to update row.
- * @throws IgniteCheckedException If failed.
- */
- public boolean updateDataRow(long link, CacheDataRow row) throws IgniteCheckedException;
-
- /**
- * @param link Row link.
- * @throws IgniteCheckedException If failed.
- */
- public void removeDataRowByLink(long link) throws IgniteCheckedException;
-
- /**
- * @param log Logger.
- */
- public void dumpStatistics(IgniteLogger log);
-}