You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2017/09/22 11:21:40 UTC
[01/35] ignite git commit: IGNITE-6342 Fixed exchange hanging on
eviction when partition is lost
Repository: ignite
Updated Branches:
refs/heads/ignite-3479 666a6accc -> db7534b43
IGNITE-6342 Fixed exchange hanging on eviction when partition is lost
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/07f5b0ce
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/07f5b0ce
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/07f5b0ce
Branch: refs/heads/ignite-3479
Commit: 07f5b0cec3f46a713e61323300f25081eb6fa258
Parents: 2ce6de3
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Thu Sep 21 16:37:42 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 21 16:37:42 2017 +0300
----------------------------------------------------------------------
.../discovery/GridDiscoveryManager.java | 18 +++---
.../dht/GridDhtPartitionTopologyImpl.java | 13 +++-
.../GridDhtPartitionsExchangeFuture.java | 2 +-
.../preloader/GridDhtPartitionsFullMessage.java | 2 +-
.../persistence/GridCacheOffheapManager.java | 2 +-
.../IgniteCacheDatabaseSharedManager.java | 2 +-
.../cache/persistence/tree/io/IOVersions.java | 3 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 3 +-
.../IgnitePdsContinuousRestartTest.java | 45 ++++++++++++-
...ousRestartTestWithSharedGroupAndIndexes.java | 67 ++++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
11 files changed, 144 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index eefec08..33350de 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -1261,7 +1261,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
"(all nodes in topology should have identical value) " +
"[locPreferIpV4=" + locPreferIpV4 + ", rmtPreferIpV4=" + rmtPreferIpV4 +
", locId8=" + U.id8(locNode.id()) + ", rmtId8=" + U.id8(n.id()) +
- ", rmtAddrs=" + U.addressesAsString(n) + ']',
+ ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]",
"Local and remote 'java.net.preferIPv4Stack' system properties do not match.");
ipV4Warned = true;
@@ -1276,7 +1276,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
throw new IgniteCheckedException("Remote node has deployment mode different from local " +
"[locId8=" + U.id8(locNode.id()) + ", locMode=" + locMode +
", rmtId8=" + U.id8(n.id()) + ", rmtMode=" + rmtMode +
- ", rmtAddrs=" + U.addressesAsString(n) + ']');
+ ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]");
boolean rmtP2pEnabled = n.attribute(ATTR_PEER_CLASSLOADING);
@@ -1284,7 +1284,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
throw new IgniteCheckedException("Remote node has peer class loading enabled flag different from" +
" local [locId8=" + U.id8(locNode.id()) + ", locPeerClassLoading=" + locP2pEnabled +
", rmtId8=" + U.id8(n.id()) + ", rmtPeerClassLoading=" + rmtP2pEnabled +
- ", rmtAddrs=" + U.addressesAsString(n) + ']');
+ ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]");
}
Boolean rmtMarshUseDfltSuid = n.attribute(ATTR_MARSHALLER_USE_DFLT_SUID);
@@ -1298,7 +1298,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
"[locMarshUseDfltSuid=" + locMarshUseDfltSuid + ", rmtMarshUseDfltSuid=" + rmtMarshUseDfltSuid +
", locNodeAddrs=" + U.addressesAsString(locNode) +
", rmtNodeAddrs=" + U.addressesAsString(n) +
- ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
+ ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ", rmtNode=" + U.toShortString(n) + "]");
}
Boolean rmtMarshStrSerVer2 = n.attribute(ATTR_MARSHALLER_USE_BINARY_STRING_SER_VER_2);
@@ -1312,7 +1312,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
"[locMarshStrSerVer2=" + locMarshStrSerVer2 + ", rmtMarshStrSerVer2=" + rmtMarshStrSerVer2 +
", locNodeAddrs=" + U.addressesAsString(locNode) +
", rmtNodeAddrs=" + U.addressesAsString(n) +
- ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
+ ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ", rmtNode=" + U.toShortString(n) + "]");
}
boolean rmtLateAssign = n.attribute(ATTR_LATE_AFFINITY_ASSIGNMENT);
@@ -1323,7 +1323,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
", locDelayAssign=" + locDelayAssign +
", rmtId8=" + U.id8(n.id()) +
", rmtLateAssign=" + rmtLateAssign +
- ", rmtAddrs=" + U.addressesAsString(n) + ']');
+ ", rmtAddrs=" + U.addressesAsString(n) + ", rmtNode=" + U.toShortString(n) + "]");
}
Boolean rmtSrvcCompatibilityEnabled = n.attribute(ATTR_SERVICES_COMPATIBILITY_MODE);
@@ -1337,7 +1337,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
", rmtSrvcCompatibilityEnabled=" + rmtSrvcCompatibilityEnabled +
", locNodeAddrs=" + U.addressesAsString(locNode) +
", rmtNodeAddrs=" + U.addressesAsString(n) +
- ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
+ ", locNodeId=" + locNode.id() + ", rmtNode=" + U.toShortString(n) + "]");
}
if (n.version().compareToIgnoreTimestamp(SERVICE_PERMISSIONS_SINCE) >= 0
@@ -1354,7 +1354,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
", rmtSecurityCompatibilityEnabled=" + rmtSecurityCompatibilityEnabled +
", locNodeAddrs=" + U.addressesAsString(locNode) +
", rmtNodeAddrs=" + U.addressesAsString(n) +
- ", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ']');
+ ", locNodeId=" + locNode.id() + ", rmtNode=" + U.toShortString(n) + "]");
}
}
@@ -1368,7 +1368,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
", locNodeAddrs=" + U.addressesAsString(locNode) +
", rmtNodeAddrs=" + U.addressesAsString(n) +
", locNodeId=" + locNode.id() + ", rmtNodeId=" + n.id() + ", " +
- ", rmtNodeVer" + n.version() + ']');
+ ", rmtNodeVer" + n.version() + ", rmtNode=" + U.toShortString(n) + "]");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
index 01d972d..0f96b5b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtPartitionTopologyImpl.java
@@ -1786,11 +1786,22 @@ public class GridDhtPartitionTopologyImpl implements GridDhtPartitionTopology {
for (Integer part : lost) {
long updSeq = updateSeq.incrementAndGet();
- GridDhtLocalPartition locPart = localPartition(part, resTopVer, false);
+ GridDhtLocalPartition locPart = localPartition(part, resTopVer, false, true);
if (locPart != null) {
boolean marked = plc == PartitionLossPolicy.IGNORE ? locPart.own() : locPart.markLost();
+ if (!marked && locPart.state() == RENTING)
+ try {
+ //TODO https://issues.apache.org/jira/browse/IGNITE-6433
+ locPart.tryEvict();
+ locPart.rent(false).get();
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to wait for RENTING partition eviction after partition LOST event",
+ e);
+ }
+
if (marked)
updateLocal(locPart.id(), locPart.state(), updSeq, resTopVer);
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 9589f09..9c7451e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1415,7 +1415,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
cacheCtx.continuousQueries().flushBackupQueue(res);
}
- }
+ }
if (err == null) {
if (centralizedAff) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
index edbfc23..4a449d1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsFullMessage.java
@@ -49,7 +49,7 @@ public class GridDhtPartitionsFullMessage extends GridDhtPartitionsAbstractMessa
/** */
private static final long serialVersionUID = 0L;
- /** */
+ /** grpId -> FullMap */
@GridToStringInclude
@GridDirectTransient
private Map<Integer, GridDhtPartitionFullMap> parts;
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 6c8d090..5c91a4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -197,7 +197,7 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
else {
// localPartition will not acquire writeLock here because create=false.
GridDhtLocalPartition part = grp.topology().localPartition(store.partId(),
- AffinityTopologyVersion.NONE, false);
+ AffinityTopologyVersion.NONE, false, true);
if (part != null && part.state() != GridDhtPartitionState.EVICTED)
state = part.state();
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
index 0dd0f6a..6ea6eff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/IgniteCacheDatabaseSharedManager.java
@@ -735,7 +735,7 @@ public class IgniteCacheDatabaseSharedManager extends GridCacheSharedManagerAdap
/** {@inheritDoc} */
@Override public boolean checkpointLockIsHeldByThread() {
- return false;
+ return true;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
index 48b0da1..d74d344 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/IOVersions.java
@@ -79,6 +79,9 @@ public final class IOVersions<V extends PageIO> {
* @return IO.
*/
public V forVersion(int ver) {
+ if (ver == 0)
+ throw new IllegalStateException("Failed to get page IO instance (page content is corrupted)");
+
return vers[ver - 1];
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
index 9433ffb..60f9d4e 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/ServerImpl.java
@@ -930,7 +930,8 @@ class ServerImpl extends TcpDiscoveryImpl {
" node is configured to use loopback address, but " + secondNode + " node is not " +
"(consider changing 'localAddress' configuration parameter) " +
"[locNodeAddrs=" + U.addressesAsString(locNode) + ", rmtNodeAddrs=" +
- U.addressesAsString(msg.addresses(), msg.hostNames()) + ']');
+ U.addressesAsString(msg.addresses(), msg.hostNames()) +
+ ", creatorNodeId=" + msg.creatorNodeId() + ']');
}
else
LT.warn(log, "Node has not been connected to topology and will repeat join process. " +
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
index 238e1d1..df62f3a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTest.java
@@ -22,6 +22,7 @@ import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.Callable;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.ignite.Ignite;
import org.apache.ignite.IgniteCache;
@@ -54,6 +55,25 @@ public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
/** */
public static final String CACHE_NAME = "cache1";
+ /** Checkpoint delay. */
+ private volatile int checkpointDelay = -1;
+
+ /** */
+ private boolean cancel = false;
+
+ /**
+ * Default constructor.
+ */
+ public IgnitePdsContinuousRestartTest() {
+ }
+
+ /**
+ * @param cancel Cancel.
+ */
+ public IgnitePdsContinuousRestartTest(boolean cancel) {
+ this.cancel = cancel;
+ }
+
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
IgniteConfiguration cfg = super.getConfiguration(gridName);
@@ -84,6 +104,7 @@ public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
cfg.setPersistentStoreConfiguration(
new PersistentStoreConfiguration()
.setWalMode(WALMode.LOG_ONLY)
+ .setCheckpointingFrequency(checkpointDelay)
);
return cfg;
@@ -195,6 +216,27 @@ public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
}
/**
+ *
+ * @throws Exception if failed.
+ */
+ public void testRebalncingDuringLoad_10_10_1_1() throws Exception {
+ checkRebalancingDuringLoad(10, 10, 1, 1);
+ }
+
+ /**
+ *
+ * @throws Exception if failed.
+ */
+ public void testRebalncingDuringLoad_10_500_8_16() throws Exception {
+ checkRebalancingDuringLoad(10, 500, 8, 16);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return TimeUnit.MINUTES.toMillis(3);
+ }
+
+ /**
* @throws Exception if failed.
*/
private void checkRebalancingDuringLoad(
@@ -203,6 +245,7 @@ public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
int threads,
final int batch
) throws Exception {
+ this.checkpointDelay = checkpointDelay;
startGrids(GRID_CNT);
@@ -245,7 +288,7 @@ public class IgnitePdsContinuousRestartTest extends GridCommonAbstractTest {
while (System.currentTimeMillis() < end) {
int idx = rnd.nextInt(GRID_CNT - 1) + 1;
- stopGrid(idx);
+ stopGrid(idx, cancel);
U.sleep(restartDelay);
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.java
new file mode 100644
index 0000000..110e677
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence;
+
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+
+/**
+ * Adding shared group and indexes to testing. It would impact how we evict partitions.
+ */
+public class IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes extends IgnitePdsContinuousRestartTest {
+ /** Ip finder. */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Cache 2 singleton group name. */
+ public static final String CACHE_GROUP_NAME = "Group2";
+
+ /**
+ * Default constructor.
+ */
+ public IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes() {
+ super(false);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoverySpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+ discoverySpi.setIpFinder(ipFinder);
+
+ CacheConfiguration ccfg2 = new CacheConfiguration();
+
+ ccfg2.setName(CACHE_NAME);
+ ccfg2.setGroupName(CACHE_GROUP_NAME);
+ ccfg2.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg2.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg2.setAffinity(new RendezvousAffinityFunction(false, 32));
+ ccfg2.setIndexedTypes(Integer.class, Integer.class);
+ ccfg2.setBackups(2);
+
+ cfg.setCacheConfiguration(ccfg2);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/07f5b0ce/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index 4f4dedf..ab8ff81 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -19,6 +19,7 @@ package org.apache.ignite.testsuites;
import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTest;
+import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsPageSizesTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePdsRecoveryAfterFileCorruptionTest;
import org.apache.ignite.internal.processors.cache.persistence.IgnitePersistenceMetricsSelfTest;
@@ -64,6 +65,8 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgnitePdsContinuousRestartTest.class);
+ suite.addTestSuite(IgnitePdsContinuousRestartTestWithSharedGroupAndIndexes.class);
+
suite.addTestSuite(IgnitePersistentStoreDataStructuresTest.class);
// Failover test
[02/35] ignite git commit: Web Console: minor refactoring.
Posted by sb...@apache.org.
Web Console: minor refactoring.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/abe4ee8f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/abe4ee8f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/abe4ee8f
Branch: refs/heads/ignite-3479
Commit: abe4ee8ff6f61ccb523dbb4b6821687302c14952
Parents: 07f5b0c
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Thu Sep 21 21:13:59 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Thu Sep 21 21:13:59 2017 +0700
----------------------------------------------------------------------
.../frontend/app/modules/agent/AgentManager.service.js | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/abe4ee8f/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
index 7c84edc..9a736b6 100644
--- a/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
+++ b/modules/web-console/frontend/app/modules/agent/AgentManager.service.js
@@ -101,7 +101,7 @@ export default class IgniteAgentManager {
this.clusterVersion = '2.1.0';
- if (!$root.IgniteDemoMode) {
+ if (!this.isDemoMode()) {
this.connectionSbj.subscribe({
next: ({cluster}) => {
const version = _.get(cluster, 'clusterVersion');
@@ -115,6 +115,10 @@ export default class IgniteAgentManager {
}
}
+ isDemoMode() {
+ return this.$root.IgniteDemoMode;
+ }
+
available(sinceVersion) {
return this.Version.since(this.clusterVersion, sinceVersion);
}
@@ -141,7 +145,7 @@ export default class IgniteAgentManager {
self.socket.on('agents:stat', ({clusters, count}) => {
const conn = self.connectionSbj.getValue();
- conn.update(self.$root.IgniteDemoMode, count, clusters);
+ conn.update(self.isDemoMode(), count, clusters);
self.connectionSbj.next(conn);
});
[20/35] ignite git commit: IGNITE-6334 Throttle writing threads
during ongoing checkpoint - Fixes #2710.
Posted by sb...@apache.org.
IGNITE-6334 Throttle writing threads during ongoing checkpoint - Fixes #2710.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f1205511
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f1205511
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f1205511
Branch: refs/heads/ignite-3479
Commit: f120551159fd2eeed8cedc0ca3e3ddc394505737
Parents: 1649c53
Author: Ivan Rakov <iv...@gmail.com>
Authored: Fri Sep 22 12:40:22 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 22 12:40:22 2017 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 5 +
.../PersistentStoreConfiguration.java | 28 +-
.../GridCacheDatabaseSharedManager.java | 48 ++-
.../persistence/pagemem/PageMemoryImpl.java | 96 +++++-
.../persistence/pagemem/PagesWriteThrottle.java | 104 ++++++
.../pagemem/BPlusTreePageMemoryImplTest.java | 4 +-
.../BPlusTreeReuseListPageMemoryImplTest.java | 3 +-
.../MetadataStoragePageMemoryImplTest.java | 4 +-
.../pagemem/PageMemoryImplNoLoadTest.java | 4 +-
.../persistence/pagemem/PageMemoryImplTest.java | 4 +-
.../pagemem/PagesWriteThrottleSandboxTest.java | 264 +++++++++++++++
.../pagemem/PagesWriteThrottleSmokeTest.java | 322 +++++++++++++++++++
.../ignite/testsuites/IgnitePdsTestSuite.java | 4 +
13 files changed, 866 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 5f1839b..39e65e1 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.Properties;
import javax.net.ssl.HostnameVerifier;
import org.apache.ignite.cluster.ClusterGroup;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
import org.apache.ignite.lang.IgnitePredicate;
import org.jetbrains.annotations.Nullable;
@@ -726,6 +727,10 @@ public final class IgniteSystemProperties {
*/
public static final String IGNITE_WAL_LOG_TX_RECORDS = "IGNITE_WAL_LOG_TX_RECORDS";
+ /** If this property is set, {@link PersistentStoreConfiguration#writeThrottlingEnabled} will be overridden to true
+ * independent of initial value in configuration. */
+ public static final String IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED = "IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED";
+
/**
* Enforces singleton.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
index abca5a5..c44e92d 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
@@ -84,6 +84,9 @@ public class PersistentStoreConfiguration implements Serializable {
/** Default wal archive directory. */
public static final String DFLT_WAL_ARCHIVE_PATH = "db/wal/archive";
+ /** Default write throttling enabled. */
+ public static final boolean DFLT_WRITE_THROTTLING_ENABLED = false;
+
/** */
private String persistenceStorePath;
@@ -162,6 +165,11 @@ public class PersistentStoreConfiguration implements Serializable {
private long walAutoArchiveAfterInactivity = -1;
/**
+ * If true, threads that generate dirty pages too fast during ongoing checkpoint will be throttled.
+ */
+ private boolean writeThrottlingEnabled = DFLT_WRITE_THROTTLING_ENABLED;
+
+ /**
* Returns a path the root directory where the Persistent Store will persist data and indexes.
*/
public String getPersistentStorePath() {
@@ -240,7 +248,7 @@ public class PersistentStoreConfiguration implements Serializable {
/**
* Sets a number of threads to use for the checkpointing purposes.
*
- * @param checkpointingThreads Number of checkpointing threads. One thread is used by default.
+ * @param checkpointingThreads Number of checkpointing threads. Four threads are used by default.
* @return {@code this} for chaining.
*/
public PersistentStoreConfiguration setCheckpointingThreads(int checkpointingThreads) {
@@ -402,6 +410,24 @@ public class PersistentStoreConfiguration implements Serializable {
}
/**
+ * Gets flag indicating whether write throttling is enabled.
+ */
+ public boolean isWriteThrottlingEnabled() {
+ return writeThrottlingEnabled;
+ }
+
+ /**
+ * Sets flag indicating whether write throttling is enabled.
+ *
+ * @param writeThrottlingEnabled Write throttling enabled flag.
+ */
+ public PersistentStoreConfiguration setWriteThrottlingEnabled(boolean writeThrottlingEnabled) {
+ this.writeThrottlingEnabled = writeThrottlingEnabled;
+
+ return this;
+ }
+
+ /**
* Gets the length of the time interval for rate-based metrics. This interval defines a window over which
* hits will be tracked. Default value is {@link #DFLT_RATE_TIME_INTERVAL_MILLIS}.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5f03d8f..62210dc 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -301,6 +301,12 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
private ObjectName persistenceMetricsMbeanName;
+ /** Counter for written checkpoint pages. Not null only if checkpoint is running. */
+ private volatile AtomicInteger writtenPagesCntr = null;
+
+ /** Number of pages in current checkpoint. */
+ private volatile int currCheckpointPagesCnt;
+
/**
* @param ctx Kernal context.
*/
@@ -667,6 +673,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
"Checkpoint page buffer size is too big, setting to an adjusted cache size [size="
+ U.readableSize(cacheSize, false) + ", memPlc=" + plcCfg.getName() + ']');
+ boolean writeThrottlingEnabled = persistenceCfg.isWriteThrottlingEnabled();
+
+ if (IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_OVERRIDE_WRITE_THROTTLING_ENABLED, false))
+ writeThrottlingEnabled = true;
+
PageMemoryImpl pageMem = new PageMemoryImpl(
memProvider,
calculateFragmentSizes(
@@ -699,7 +710,8 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
},
this,
- memMetrics
+ memMetrics,
+ writeThrottlingEnabled
);
memMetrics.pageMemory(pageMem);
@@ -942,7 +954,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
- * Gets the checkpoint read lock. While this lock is held, checkpoint thread will not acquiSnapshotWorkerre memory
+ * Gets the checkpoint read lock. While this lock is held, checkpoint thread will not acquireSnapshotWorker memory
* state.
*/
@SuppressWarnings("LockAcquiredButNotSafelyReleased")
@@ -1906,6 +1918,20 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
}
/**
+ * Counter for written checkpoint pages. Not null only if checkpoint is running.
+ */
+ public AtomicInteger writtenPagesCounter() {
+ return writtenPagesCntr;
+ }
+
+ /**
+ * @return Number of pages in current checkpoint. If checkpoint is not running, returns 0.
+ */
+ public int currentCheckpointPagesCount() {
+ return currCheckpointPagesCnt;
+ }
+
+ /**
* @param cpTs Checkpoint timestamp.
* @param cpId Checkpoint ID.
* @param type Checkpoint type.
@@ -2038,6 +2064,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
Checkpoint chp = markCheckpointBegin(tracker);
+ currCheckpointPagesCnt = chp.pagesSize;
+
+ writtenPagesCntr = new AtomicInteger();
+
boolean interrupted = true;
try {
@@ -2049,7 +2079,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
asyncRunner == null ? 1 : chp.cpPages.collectionsSize());
tracker.onPagesWriteStart();
- final AtomicInteger writtenPagesCtr = new AtomicInteger();
+
final int totalPagesToWriteCnt = chp.cpPages.size();
if (asyncRunner != null) {
@@ -2059,7 +2089,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
chp.cpPages.innerCollection(i),
updStores,
doneWriteFut,
- writtenPagesCtr,
totalPagesToWriteCnt
);
@@ -2078,7 +2107,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
chp.cpPages,
updStores,
doneWriteFut,
- writtenPagesCtr,
totalPagesToWriteCnt);
write.run();
@@ -2402,6 +2430,10 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
chp.cpEntry.checkpointMark(),
null,
CheckpointEntryType.END);
+
+ writtenPagesCntr = null;
+
+ currCheckpointPagesCnt = 0;
}
checkpointHist.onCheckpointFinished(chp);
@@ -2498,9 +2530,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
/** */
private CountDownFuture doneFut;
- /** Counter for all written pages. May be shared between several workers */
- private AtomicInteger writtenPagesCntr;
-
/** Total pages to write, counter may be greater than {@link #writePageIds} size */
private final int totalPagesToWrite;
@@ -2511,7 +2540,6 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @param writePageIds Collection of page IDs to write.
* @param updStores
* @param doneFut
- * @param writtenPagesCntr all written pages counter, may be shared between several write tasks
* @param totalPagesToWrite total pages to be written under this checkpoint
*/
private WriteCheckpointPages(
@@ -2519,13 +2547,11 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
final Collection<FullPageId> writePageIds,
final GridConcurrentHashSet<PageStore> updStores,
final CountDownFuture doneFut,
- @NotNull final AtomicInteger writtenPagesCntr,
final int totalPagesToWrite) {
this.tracker = tracker;
this.writePageIds = writePageIds;
this.updStores = updStores;
this.doneFut = doneFut;
- this.writtenPagesCntr = writtenPagesCntr;
this.totalPagesToWrite = totalPagesToWrite;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
index dbb64f8..1da17b5 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java
@@ -33,6 +33,7 @@ import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
@@ -56,6 +57,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord;
import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.persistence.CheckpointLockStateChecker;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
import org.apache.ignite.internal.processors.cache.persistence.MemoryMetricsImpl;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.TrackingPageIO;
@@ -179,6 +181,9 @@ public class PageMemoryImpl implements PageMemoryEx {
/** State checker. */
private final CheckpointLockStateChecker stateChecker;
+ /** Number of used pages in checkpoint buffer. */
+ private final AtomicInteger cpBufPagesCntr = new AtomicInteger(0);
+
/** */
private ExecutorService asyncRunner = new ThreadPoolExecutor(
0,
@@ -217,6 +222,12 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Flush dirty page closure. When possible, will be called by evictPage(). */
private final GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker;
+ /** Pages write throttle. */
+ private PagesWriteThrottle writeThrottle;
+
+ /** Write throttle enabled flag. */
+ private boolean throttleEnabled;
+
/** */
private boolean pageEvictWarned;
@@ -232,6 +243,7 @@ public class PageMemoryImpl implements PageMemoryEx {
* @param pageSize Page size.
* @param flushDirtyPage Callback invoked when a dirty page is evicted.
* @param changeTracker Callback invoked to track changes in pages.
+ * @param throttleEnabled Write throttle enabled flag.
*/
public PageMemoryImpl(
DirectMemoryProvider directMemoryProvider,
@@ -241,7 +253,8 @@ public class PageMemoryImpl implements PageMemoryEx {
GridInClosure3X<FullPageId, ByteBuffer, Integer> flushDirtyPage,
GridInClosure3X<Long, FullPageId, PageMemoryEx> changeTracker,
CheckpointLockStateChecker stateChecker,
- MemoryMetricsImpl memMetrics
+ MemoryMetricsImpl memMetrics,
+ boolean throttleEnabled
) {
assert sharedCtx != null;
@@ -253,6 +266,7 @@ public class PageMemoryImpl implements PageMemoryEx {
this.flushDirtyPage = flushDirtyPage;
this.changeTracker = changeTracker;
this.stateChecker = stateChecker;
+ this.throttleEnabled = throttleEnabled;
storeMgr = sharedCtx.pageStore();
walMgr = sharedCtx.wal();
@@ -290,7 +304,7 @@ public class PageMemoryImpl implements PageMemoryEx {
DirectMemoryRegion cpReg = regions.get(regs - 1);
- checkpointPool = new PagePool(regs - 1, cpReg);
+ checkpointPool = new PagePool(regs - 1, cpReg, cpBufPagesCntr);
long checkpointBuf = cpReg.size();
@@ -305,12 +319,14 @@ public class PageMemoryImpl implements PageMemoryEx {
totalAllocated += reg.size();
- segments[i] = new Segment(i, regions.get(i), checkpointPool.pages() / segments.length);
+ segments[i] = new Segment(i, regions.get(i), checkpointPool.pages() / segments.length, throttleEnabled);
pages += segments[i].pages();
totalTblSize += segments[i].tableSize();
}
+ initWriteThrottle();
+
if (log.isInfoEnabled())
log.info("Started page memory [memoryAllocated=" + U.readableSize(totalAllocated, false) +
", pages=" + pages +
@@ -319,6 +335,21 @@ public class PageMemoryImpl implements PageMemoryEx {
']');
}
+ /**
+ *
+ */
+ private void initWriteThrottle() {
+ if (!(sharedCtx.database() instanceof GridCacheDatabaseSharedManager)) {
+ log.error("Write throttle can't start. Unexpected class of database manager: " +
+ sharedCtx.database().getClass());
+
+ throttleEnabled = false;
+ }
+
+ if (throttleEnabled)
+ writeThrottle = new PagesWriteThrottle(this, (GridCacheDatabaseSharedManager)sharedCtx.database());
+ }
+
/** {@inheritDoc} */
@SuppressWarnings("OverlyStrongTypeCast")
@Override public void stop() throws IgniteException {
@@ -774,6 +805,18 @@ public class PageMemoryImpl implements PageMemoryEx {
return true;
}
+ /**
+ * @param dirtyRatioThreshold Throttle threshold.
+ */
+ boolean shouldThrottle(double dirtyRatioThreshold) {
+ for (Segment segment : segments) {
+ if (segment.shouldThrottle(dirtyRatioThreshold))
+ return true;
+ }
+
+ return false;
+ }
+
/** {@inheritDoc} */
@Override public GridMultiCollectionWrapper<FullPageId> beginCheckpoint() throws IgniteException {
Collection[] collections = new Collection[segments.length];
@@ -799,6 +842,9 @@ public class PageMemoryImpl implements PageMemoryEx {
@Override public void finishCheckpoint() {
for (Segment seg : segments)
seg.segCheckpointPages = null;
+
+ if (throttleEnabled)
+ writeThrottle.onFinishCheckpoint();
}
/** {@inheritDoc} */
@@ -1219,6 +1265,9 @@ public class PageMemoryImpl implements PageMemoryEx {
try {
rwLock.writeUnlock(page + PAGE_LOCK_OFFSET, PageIdUtils.tag(pageId));
+
+ if (throttleEnabled && !restore && markDirty && !dirty)
+ writeThrottle.onMarkDirty(isInCheckpoint(fullId));
}
catch (AssertionError ex) {
StringBuilder sb = new StringBuilder(sysPageSize * 2);
@@ -1310,6 +1359,20 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
+ * Number of used pages in checkpoint buffer.
+ */
+ public int checkpointBufferPagesCount() {
+ return cpBufPagesCntr.get();
+ }
+
+ /**
+ * Number of used pages in checkpoint buffer.
+ */
+ public int checkpointBufferPagesSize() {
+ return checkpointPool.pages();
+ }
+
+ /**
* This method must be called in synchronized context.
*
* @param absPtr Absolute pointer.
@@ -1385,6 +1448,9 @@ public class PageMemoryImpl implements PageMemoryEx {
/** Direct memory region. */
protected final DirectMemoryRegion region;
+ /** Pool pages counter. */
+ protected final AtomicInteger pagesCntr;
+
/** */
protected long lastAllocatedIdxPtr;
@@ -1397,10 +1463,12 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param idx Index.
* @param region Region
+ * @param pagesCntr Pages counter.
*/
- protected PagePool(int idx, DirectMemoryRegion region) {
+ protected PagePool(int idx, DirectMemoryRegion region, AtomicInteger pagesCntr) {
this.idx = idx;
this.region = region;
+ this.pagesCntr = pagesCntr;
long base = (region.address() + 7) & ~0x7;
@@ -1427,6 +1495,9 @@ public class PageMemoryImpl implements PageMemoryEx {
* @throws GridOffHeapOutOfMemoryException If failed to allocate new free page.
*/
private long borrowOrAllocateFreePage(long pageId) throws GridOffHeapOutOfMemoryException {
+ if (pagesCntr != null)
+ pagesCntr.getAndIncrement();
+
long relPtr = borrowFreePage();
return relPtr != INVALID_REL_PTR ? relPtr : allocateFreePage(pageId);
@@ -1500,6 +1571,9 @@ public class PageMemoryImpl implements PageMemoryEx {
assert !PageHeader.isAcquired(absPtr) : "Release pinned page: " + PageHeader.fullPageId(absPtr);
+ if (pagesCntr != null)
+ pagesCntr.getAndDecrement();
+
while (true) {
long freePageRelPtrMasked = GridUnsafe.getLong(freePageListPtr);
@@ -1580,8 +1654,9 @@ public class PageMemoryImpl implements PageMemoryEx {
/**
* @param region Memory region.
+ * @param throttlingEnabled Write throttling enabled flag.
*/
- private Segment(int idx, DirectMemoryRegion region, int cpPoolPages) {
+ private Segment(int idx, DirectMemoryRegion region, int cpPoolPages, boolean throttlingEnabled) {
long totalMemory = region.size();
int pages = (int)(totalMemory / sysPageSize);
@@ -1596,9 +1671,9 @@ public class PageMemoryImpl implements PageMemoryEx {
DirectMemoryRegion poolRegion = region.slice(memPerTbl + 8);
- pool = new PagePool(idx, poolRegion);
+ pool = new PagePool(idx, poolRegion, null);
- maxDirtyPages = Math.min(pool.pages() * 2 / 3, cpPoolPages);
+ maxDirtyPages = throttlingEnabled ? pool.pages() * 3 / 4 : Math.min(pool.pages() * 2 / 3, cpPoolPages);
}
/**
@@ -1609,6 +1684,13 @@ public class PageMemoryImpl implements PageMemoryEx {
}
/**
+ * @param dirtyRatioThreshold Throttle threshold.
+ */
+ private boolean shouldThrottle(double dirtyRatioThreshold) {
+ return ((double)dirtyPages.size()) / pages() > dirtyRatioThreshold;
+ }
+
+ /**
* @return Max number of pages this segment can allocate.
*/
private int pages() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
new file mode 100644
index 0000000..d0c67c7
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottle.java
@@ -0,0 +1,104 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+
+/**
+ * Throttles threads that generate dirty pages during ongoing checkpoint.
+ * Designed to avoid zero dropdowns that can happen if checkpoint buffer is overflowed.
+ */
+public class PagesWriteThrottle {
+ /** Page memory. */
+ private final PageMemoryImpl pageMemory;
+
+ /** Database manager. */
+ private final GridCacheDatabaseSharedManager dbSharedMgr;
+
+ /** Starting throttle time. Limits write speed to 1000 MB/s. */
+ private static final long STARTING_THROTTLE_NANOS = 4000;
+
+ /** Backoff ratio. Each next park will be this times longer. */
+ private static final double BACKOFF_RATIO = 1.05;
+
+ /** Exponential backoff counter. */
+ private final AtomicInteger exponentialBackoffCntr = new AtomicInteger(0);
+ /**
+ * @param pageMemory Page memory.
+ * @param dbSharedMgr Database manager.
+ */
+ public PagesWriteThrottle(PageMemoryImpl pageMemory, GridCacheDatabaseSharedManager dbSharedMgr) {
+ this.pageMemory = pageMemory;
+ this.dbSharedMgr = dbSharedMgr;
+ }
+
+ /**
+ *
+ */
+ public void onMarkDirty(boolean isInCheckpoint) {
+ assert dbSharedMgr.checkpointLockIsHeldByThread();
+
+ AtomicInteger writtenPagesCntr = dbSharedMgr.writtenPagesCounter();
+
+ if (writtenPagesCntr == null)
+ return; // Don't throttle if checkpoint is not running.
+
+ boolean shouldThrottle = false;
+
+ if (isInCheckpoint) {
+ int checkpointBufLimit = pageMemory.checkpointBufferPagesSize() * 2 / 3;
+
+ shouldThrottle = pageMemory.checkpointBufferPagesCount() > checkpointBufLimit;
+ }
+
+ if (!shouldThrottle) {
+ int cpWrittenPages = writtenPagesCntr.get();
+
+ int cpTotalPages = dbSharedMgr.currentCheckpointPagesCount();
+
+ if (cpWrittenPages == cpTotalPages) {
+ // Checkpoint is already in fsync stage, increasing maximum ratio of dirty pages to 3/4
+ shouldThrottle = pageMemory.shouldThrottle(3.0 / 4);
+ } else {
+ double dirtyRatioThreshold = ((double)cpWrittenPages) / cpTotalPages;
+
+ // Starting with 0.05 to avoid throttle right after checkpoint start
+ // 7/12 is maximum ratio of dirty pages
+ dirtyRatioThreshold = (dirtyRatioThreshold * 0.95 + 0.05) * 7 / 12;
+
+ shouldThrottle = pageMemory.shouldThrottle(dirtyRatioThreshold);
+ }
+ }
+
+ if (shouldThrottle) {
+ int throttleLevel = exponentialBackoffCntr.getAndIncrement();
+
+ LockSupport.parkNanos((long)(STARTING_THROTTLE_NANOS * Math.pow(BACKOFF_RATIO, throttleLevel)));
+ }
+ else
+ exponentialBackoffCntr.set(0);
+ }
+
+ /**
+ *
+ */
+ public void onFinishCheckpoint() {
+ exponentialBackoffCntr.set(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
index 6f58782..56d09f8 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
@@ -82,7 +82,9 @@ public class BPlusTreePageMemoryImplTest extends BPlusTreeSelfTest {
return true;
}
},
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()),
+ false
+ );
mem.start();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
index b263d4f..39183b2 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
@@ -82,7 +82,8 @@ public class BPlusTreeReuseListPageMemoryImplTest extends BPlusTreeReuseSelfTest
return true;
}
},
- new MemoryMetricsImpl(new MemoryPolicyConfiguration())
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()),
+ false
);
mem.start();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
index d9257bd..a427c63 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
@@ -97,6 +97,8 @@ public class MetadataStoragePageMemoryImplTest extends MetadataStorageSelfTest{
return true;
}
},
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()),
+ false
+ );
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
index 1fff1f0..467ede4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
@@ -88,7 +88,9 @@ public class PageMemoryImplNoLoadTest extends PageMemoryNoLoadSelfTest {
return true;
}
},
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()),
+ false
+ );
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
index 0366eca..c5997fa 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
@@ -110,7 +110,9 @@ public class PageMemoryImplTest extends GridCommonAbstractTest {
return true;
}
},
- new MemoryMetricsImpl(new MemoryPolicyConfiguration()));
+ new MemoryMetricsImpl(new MemoryPolicyConfiguration()),
+ false
+ );
mem.start();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSandboxTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSandboxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSandboxTest.java
new file mode 100644
index 0000000..409ab84
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSandboxTest.java
@@ -0,0 +1,264 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.Serializable;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.MemoryMetrics;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
+import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ * Test to visualize and debug {@link PagesWriteThrottle}.
+ * Prints puts/gets rate, number of dirty pages, pages written in current checkpoint and pages in checkpoint buffer.
+ * Not intended to be part of any test suite.
+ */
+public class PagesWriteThrottleSandboxTest extends GridCommonAbstractTest {
+ /** Ip finder. */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache1";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoverySpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+ discoverySpi.setIpFinder(ipFinder);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
+ .setMaxSize(4000L * 1024 * 1024)
+ .setName("dfltMemPlc")
+ .setMetricsEnabled(true));
+
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg1 = new CacheConfiguration();
+
+ ccfg1.setName(CACHE_NAME);
+ ccfg1.setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ ccfg1.setAffinity(new RendezvousAffinityFunction(false, 64));
+
+ cfg.setCacheConfiguration(ccfg1);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.BACKGROUND)
+ .setCheckpointingFrequency(20_000)
+ .setCheckpointingPageBufferSize(1000L * 1000 * 1000)
+ .setWriteThrottlingEnabled(true));
+
+ cfg.setConsistentId(gridName);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 100 * 60 * 1000;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testThrottle() throws Exception {
+ startGrids(1).active(true);
+
+ try {
+ final Ignite ig = ignite(0);
+
+ final int keyCnt = 4_000_000;
+
+ final AtomicBoolean run = new AtomicBoolean(true);
+
+ final HitRateMetrics getRate = new HitRateMetrics(5000, 5);
+
+ GridTestUtils.runMultiThreadedAsync(new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ while (run.get()) {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ int key = rnd.nextInt(keyCnt * 2);
+
+ ignite(0).cache(CACHE_NAME).get(key);
+
+ getRate.onHit();
+ }
+
+ return null;
+ }
+ }, 2, "read-loader");
+
+ final HitRateMetrics putRate = new HitRateMetrics(1000, 5);
+
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ while (run.get()) {
+ long dirtyPages = 0;
+
+ for (MemoryMetrics m : ig.memoryMetrics())
+ if (m.getName().equals("dfltMemPlc"))
+ dirtyPages = m.getDirtyPages();
+
+ long cpBufPages = 0;
+
+ long cpWrittenPages;
+
+ AtomicInteger cntr = ((GridCacheDatabaseSharedManager)(((IgniteEx)ignite(0))
+ .context().cache().context().database())).writtenPagesCounter();
+
+ cpWrittenPages = cntr == null ? 0 : cntr.get();
+
+ try {
+ cpBufPages = ((PageMemoryImpl)((IgniteEx)ignite(0)).context().cache().context().database()
+ .memoryPolicy("dfltMemPlc").pageMemory()).checkpointBufferPagesCount();
+ }
+ catch (IgniteCheckedException e) {
+ e.printStackTrace();
+ }
+
+ System.out.println("@@@ putsPerSec=," + (putRate.getRate()) + ", getsPerSec=," + (getRate.getRate()) + ", dirtyPages=," + dirtyPages + ", cpWrittenPages=," + cpWrittenPages +", cpBufPages=," + cpBufPages);
+
+ try {
+ Thread.sleep(1000);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }, "metrics-view");
+
+ try (IgniteDataStreamer<Object, Object> ds = ig.dataStreamer(CACHE_NAME)) {
+ ds.allowOverwrite(true);
+
+ for (int i = 0; i < keyCnt * 10; i++) {
+ ds.addData(ThreadLocalRandom.current().nextInt(keyCnt), new TestValue(ThreadLocalRandom.current().nextInt(),
+ ThreadLocalRandom.current().nextInt()));
+
+ putRate.onHit();
+ }
+ }
+
+ run.set(false);
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestValue implements Serializable {
+ /** */
+ private final int v1;
+
+ /** */
+ private final int v2;
+
+ /** */
+ private byte[] payload = new byte[400 + ThreadLocalRandom.current().nextInt(20)];
+
+ /**
+ * @param v1 Value 1.
+ * @param v2 Value 2.
+ */
+ private TestValue(int v1, int v2) {
+ this.v1 = v1;
+ this.v2 = v2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ TestValue val = (TestValue)o;
+
+ return v1 == val.v1 && v2 == val.v2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = v1;
+
+ res = 31 * res + v2;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(TestValue.class, this);
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "snapshot", false));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
new file mode 100644
index 0000000..12a601d
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
@@ -0,0 +1,322 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.ignite.internal.processors.cache.persistence.pagemem;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.LockSupport;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cache.CacheAtomicityMode;
+import org.apache.ignite.cache.CacheWriteSynchronizationMode;
+import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIODecorator;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
+import org.apache.ignite.internal.processors.cache.ratemetrics.HitRateMetrics;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+/**
+ *
+ */
+public class PagesWriteThrottleSmokeTest extends GridCommonAbstractTest {
+ /** Ip finder. */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** Slow checkpoint enabled. */
+ private final AtomicBoolean slowCheckpointEnabled = new AtomicBoolean(true);
+
+ /** Cache name. */
+ private static final String CACHE_NAME = "cache1";
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ TcpDiscoverySpi discoverySpi = (TcpDiscoverySpi)cfg.getDiscoverySpi();
+ discoverySpi.setIpFinder(ipFinder);
+
+ MemoryConfiguration dbCfg = new MemoryConfiguration();
+
+ dbCfg.setMemoryPolicies(new MemoryPolicyConfiguration()
+ .setMaxSize(400 * 1024 * 1024)
+ .setName("dfltMemPlc")
+ .setMetricsEnabled(true));
+
+ dbCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ cfg.setMemoryConfiguration(dbCfg);
+
+ CacheConfiguration ccfg1 = new CacheConfiguration();
+
+ ccfg1.setName(CACHE_NAME);
+ ccfg1.setAtomicityMode(CacheAtomicityMode.ATOMIC);
+ ccfg1.setWriteSynchronizationMode(CacheWriteSynchronizationMode.PRIMARY_SYNC);
+ ccfg1.setAffinity(new RendezvousAffinityFunction(false, 64));
+
+ cfg.setCacheConfiguration(ccfg1);
+
+ cfg.setPersistentStoreConfiguration(
+ new PersistentStoreConfiguration()
+ .setWalMode(WALMode.BACKGROUND)
+ .setCheckpointingFrequency(20_000)
+ .setCheckpointingPageBufferSize(200 * 1000 * 1000)
+ .setWriteThrottlingEnabled(true)
+ .setCheckpointingThreads(1)
+ .setFileIOFactory(new SlowCheckpointFileIOFactory()));
+
+ cfg.setConsistentId(gridName);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ deleteWorkFiles();
+
+ slowCheckpointEnabled.set(true);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ deleteWorkFiles();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected long getTestTimeout() {
+ return 6 * 60 * 1000;
+ }
+
+ /**
+ * @throws Exception if failed.
+ */
+ public void testThrottle() throws Exception {
+ startGrids(2).active(true);
+
+ try {
+ Ignite ig = ignite(0);
+
+ final int keyCnt = 2_000_000;
+
+ final AtomicBoolean run = new AtomicBoolean(true);
+
+ final AtomicBoolean zeroDropdown = new AtomicBoolean(false);
+
+ final HitRateMetrics putRate10secs = new HitRateMetrics(10_000, 20);
+
+ final HitRateMetrics putRate1sec = new HitRateMetrics(1_000, 20);
+
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ Thread.sleep(5000);
+
+ while (run.get()) {
+ System.out.println(
+ "Put rate over last 10 seconds: " + (putRate10secs.getRate() / 10) +
+ " puts/sec, over last 1 second: " + putRate1sec.getRate());
+
+ if (putRate10secs.getRate() == 0) {
+ zeroDropdown.set(true);
+
+ run.set(false);
+ }
+
+ Thread.sleep(1000);
+ }
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ finally {
+ run.set(false);
+ }
+ }
+ }, "rate-checker");
+
+ final IgniteCache<Integer, TestValue> cache = ig.getOrCreateCache(CACHE_NAME);
+
+ GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ long startTs = System.currentTimeMillis();
+
+ for (int i = 0; i < keyCnt * 10 && System.currentTimeMillis() - startTs < 3 * 60 * 1000; i++) {
+ if (!run.get())
+ break;
+
+ cache.put(ThreadLocalRandom.current().nextInt(keyCnt), new TestValue(ThreadLocalRandom.current().nextInt(),
+ ThreadLocalRandom.current().nextInt()));
+
+ putRate10secs.onHit();
+
+ putRate1sec.onHit();
+ }
+
+ run.set(false);
+ }
+ }, "loader");
+
+ while (run.get())
+ LockSupport.parkNanos(10_000);
+
+ if (zeroDropdown.get()) {
+ slowCheckpointEnabled.set(false);
+
+ IgniteInternalFuture cpFut1 = ((IgniteEx)ignite(0)).context().cache().context().database()
+ .wakeupForCheckpoint("test");
+
+ IgniteInternalFuture cpFut2 = ((IgniteEx)ignite(1)).context().cache().context().database()
+ .wakeupForCheckpoint("test");
+
+ cpFut1.get();
+
+ cpFut2.get();
+
+ fail("Put rate degraded to zero for at least 10 seconds");
+ }
+ }
+ finally {
+ stopAllGrids();
+ }
+ }
+
+ /**
+ *
+ */
+ private static class TestValue implements Serializable {
+ /** */
+ private final int v1;
+
+ /** */
+ private final int v2;
+
+ /** */
+ private byte[] payload = new byte[400 + ThreadLocalRandom.current().nextInt(20)];
+
+ /**
+ * @param v1 Value 1.
+ * @param v2 Value 2.
+ */
+ private TestValue(int v1, int v2) {
+ this.v1 = v1;
+ this.v2 = v2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+
+ if (o == null || getClass() != o.getClass())
+ return false;
+
+ TestValue val = (TestValue)o;
+
+ return v1 == val.v1 && v2 == val.v2;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+ int res = v1;
+
+ res = 31 * res + v2;
+
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(TestValue.class, this);
+ }
+ }
+
+ /**
+ * @throws IgniteCheckedException If failed.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "snapshot", false));
+ }
+
+ /**
+ * Create File I/O that emulates poor checkpoint write speed.
+ */
+ private class SlowCheckpointFileIOFactory implements FileIOFactory {
+ /** Serial version uid. */
+ private static final long serialVersionUID = 0L;
+
+ /** Delegate factory. */
+ private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return create(file, "rw");
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, String mode) throws IOException {
+ final FileIO delegate = delegateFactory.create(file, mode);
+
+ return new FileIODecorator(delegate) {
+ @Override public int write(ByteBuffer srcBuf) throws IOException {
+ if (slowCheckpointEnabled.get() && Thread.currentThread().getName().contains("checkpoint"))
+ LockSupport.parkNanos(5_000_000);
+
+ return delegate.write(srcBuf);
+ }
+
+ @Override public int write(ByteBuffer srcBuf, long position) throws IOException {
+ if (slowCheckpointEnabled.get() && Thread.currentThread().getName().contains("checkpoint"))
+ LockSupport.parkNanos(5_000_000);
+
+ return delegate.write(srcBuf, position);
+ }
+
+ @Override public void write(byte[] buf, int off, int len) throws IOException {
+ if (slowCheckpointEnabled.get() && Thread.currentThread().getName().contains("checkpoint"))
+ LockSupport.parkNanos(5_000_000);
+
+ delegate.write(buf, off, len);
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f1205511/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
index b2a1f65..ef7682f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite.java
@@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.cache.persistence.pagemem.BPlusTree
import org.apache.ignite.internal.processors.cache.persistence.pagemem.MetadataStoragePageMemoryImplTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplNoLoadTest;
import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImplTest;
+import org.apache.ignite.internal.processors.cache.persistence.pagemem.PagesWriteThrottleSmokeTest;
import org.apache.ignite.internal.processors.database.IgniteDbClientNearCachePutGetTest;
import org.apache.ignite.internal.processors.database.IgniteDbDynamicCacheSelfTest;
import org.apache.ignite.internal.processors.database.IgniteDbMultiNodePutGetTest;
@@ -80,6 +81,9 @@ public class IgnitePdsTestSuite extends TestSuite {
suite.addTestSuite(DefaultPageSizeBackwardsCompatibilityTest.class);
+ // Write throttling
+ suite.addTestSuite(PagesWriteThrottleSmokeTest.class);
+
return suite;
}
}
[04/35] ignite git commit: IGNITE-6465: JDBC thin driver: added
SQLSTATE propagation for BatchUpdateException. This closes #2719.
Posted by sb...@apache.org.
IGNITE-6465: JDBC thin driver: added SQLSTATE propagation for BatchUpdateException. This closes #2719.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f8ae8f9b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f8ae8f9b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f8ae8f9b
Branch: refs/heads/ignite-3479
Commit: f8ae8f9bcfe28a0bb558ea7c4bb97e484fb35ae3
Parents: c7d591f
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Thu Sep 21 17:51:26 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Thu Sep 21 17:51:26 2017 +0300
----------------------------------------------------------------------
.../jdbc/thin/JdbcThinErrorsSelfTest.java | 31 ++++++++++++++++++++
.../internal/jdbc/thin/JdbcThinStatement.java | 7 +++--
2 files changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8ae8f9b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
index afd06ed..db70f3be 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinErrorsSelfTest.java
@@ -17,9 +17,11 @@
package org.apache.ignite.jdbc.thin;
+import java.sql.BatchUpdateException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
+import java.sql.Statement;
import org.apache.ignite.jdbc.JdbcErrorsAbstractSelfTest;
import org.apache.ignite.lang.IgniteCallable;
@@ -74,4 +76,33 @@ public class JdbcThinErrorsSelfTest extends JdbcErrorsAbstractSelfTest {
}
}, "0700E");
}
+
+ /**
+ * Test error code for the case when error is caused on batch execution.
+ * @throws SQLException if failed.
+ */
+ @SuppressWarnings("MagicConstant")
+ public void testBatchUpdateException() throws SQLException {
+ try (final Connection conn = getConnection()) {
+ try (Statement stmt = conn.createStatement()) {
+ stmt.executeUpdate("CREATE TABLE test (id int primary key, val varchar)");
+
+ stmt.addBatch("insert into test (id, val) values (1, 'val1')");
+ stmt.addBatch("insert into test (id, val) values (2, 'val2')");
+ stmt.addBatch("insert into test (id1, val1) values (3, 'val3')");
+
+ stmt.executeBatch();
+
+ fail("BatchUpdateException is expected");
+ }
+ catch (BatchUpdateException e) {
+ assertEquals(2, e.getUpdateCounts().length);
+
+ for (int updCnt : e.getUpdateCounts())
+ assertEquals(1, updCnt);
+
+ assertEquals("42000", e.getSQLState());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f8ae8f9b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
index 6ab50de..8e096c8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinStatement.java
@@ -27,6 +27,7 @@ import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.cache.query.SqlQuery;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.odbc.SqlStateCode;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.apache.ignite.internal.processors.odbc.jdbc.JdbcBatchExecuteRequest;
@@ -371,8 +372,10 @@ public class JdbcThinStatement implements Statement {
try {
JdbcBatchExecuteResult res = conn.sendRequest(new JdbcBatchExecuteRequest(conn.getSchema(), batch));
- if (res.errorCode() != ClientListenerResponse.STATUS_SUCCESS)
- throw new BatchUpdateException(res.errorMessage(), null, res.errorCode(), res.updateCounts());
+ if (res.errorCode() != ClientListenerResponse.STATUS_SUCCESS) {
+ throw new BatchUpdateException(res.errorMessage(), IgniteQueryErrorCode.codeToSqlState(res.errorCode()),
+ res.errorCode(), res.updateCounts());
+ }
return res.updateCounts();
}
[06/35] ignite git commit: ignite-6470 Fixed wrong casting og long
value to int
Posted by sb...@apache.org.
ignite-6470 Fixed wrong casting og long value to int
Signed-off-by: Andrey Gura <ag...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b98ffab7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b98ffab7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b98ffab7
Branch: refs/heads/ignite-3479
Commit: b98ffab7b35eafe97c544b141dd701f4971e16e1
Parents: e8379ae
Author: EdShangGG <es...@gridgain.com>
Authored: Thu Sep 21 18:36:37 2017 +0300
Committer: Andrey Gura <ag...@apache.org>
Committed: Thu Sep 21 18:36:37 2017 +0300
----------------------------------------------------------------------
.../internal/processors/cache/persistence/file/FilePageStore.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b98ffab7/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 98764a2..0547dbc 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -538,6 +538,6 @@ public class FilePageStore implements PageStore {
if (!inited)
return 0;
- return (int)(allocated.get() - headerSize()) / pageSize;
+ return (int)((allocated.get() - headerSize()) / pageSize);
}
}
[08/35] ignite git commit: Merge remote-tracking branch
'origin/master'
Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ea0dcc46
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ea0dcc46
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ea0dcc46
Branch: refs/heads/ignite-3479
Commit: ea0dcc460bfc4d26e52c8f5760b6c3a6e7e9c351
Parents: 37b4b1d b98ffab
Author: devozerov <pp...@gmail.com>
Authored: Thu Sep 21 18:53:46 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Thu Sep 21 18:53:46 2017 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 5 +
.../PersistentStoreConfiguration.java | 6 +-
.../GridCacheDatabaseSharedManager.java | 7 +-
.../cache/persistence/file/AsyncFileIO.java | 218 +++++++++++++++++++
.../persistence/file/AsyncFileIOFactory.java | 52 +++++
.../cache/persistence/file/FileIOFactory.java | 25 ++-
.../cache/persistence/file/FilePageStore.java | 7 +-
.../file/FileVersionCheckingFactory.java | 2 +-
.../persistence/file/RandomAccessFileIO.java | 48 ++--
.../file/RandomAccessFileIOFactory.java | 14 +-
.../wal/AbstractWalRecordsIterator.java | 2 +-
.../cache/persistence/wal/FileInput.java | 7 +
.../wal/FileWriteAheadLogManager.java | 8 +-
.../reader/StandaloneWalRecordsIterator.java | 4 +-
.../internal/util/future/GridFutureAdapter.java | 16 ++
.../resources/META-INF/classnames.properties | 2 +
.../file/IgnitePdsThreadInterruptionTest.java | 205 +++++++++++++++++
.../db/wal/IgniteWalFlushFailoverTest.java | 22 +-
.../db/wal/crc/IgniteDataIntegrityTests.java | 39 ++--
.../development/utils/IgniteWalConverter.java | 1 -
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 +
21 files changed, 607 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
[09/35] ignite git commit: IGNITE-6460 Fixed consistentId used in
lightweight ClusterNode. Fixes #2707
Posted by sb...@apache.org.
IGNITE-6460 Fixed consistentId used in lightweight ClusterNode. Fixes #2707
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/db96b790
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/db96b790
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/db96b790
Branch: refs/heads/ignite-3479
Commit: db96b79008cd63fa8fab17c120ee0d71162c2c78
Parents: ea0dcc4
Author: Eduard Shangareev <ed...@gmail.com>
Authored: Thu Sep 21 18:55:00 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 21 18:56:40 2017 +0300
----------------------------------------------------------------------
.../ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/db96b790/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index b2797a40..20fb6c5 100644
--- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -652,7 +652,7 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
/**
* IMPORTANT!
- * Only purpose of this constructor is creating node which contains only necessary data to store on disc
+ * Only purpose of this constructor is creating node which contains necessary data to store on disc only
* @param node to copy data from
*/
public TcpDiscoveryNode(
@@ -667,6 +667,6 @@ public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements Cluste
this.daemon = node.isDaemon();
this.clientRouterNodeId = node.isClient() ? node.id() : null;
- attrs = Collections.emptyMap();
+ attrs = Collections.singletonMap(ATTR_NODE_CONSISTENT_ID, consistentId);
}
}
[33/35] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-3478
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-3478
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3ad260d2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3ad260d2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3ad260d2
Branch: refs/heads/ignite-3479
Commit: 3ad260d24575321650570c7e8136b69c5d5bf02f
Parents: f6b0148 1f2f872
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 14:17:35 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 14:17:35 2017 +0300
----------------------------------------------------------------------
README.md | 149 +++--
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 3 +
.../jdbc/thin/JdbcThinErrorsSelfTest.java | 31 +
.../thin/JdbcThinSelectAfterAlterTable.java | 173 +++++
.../apache/ignite/IgniteSystemProperties.java | 10 +
.../PersistentStoreConfiguration.java | 34 +-
.../internal/GridEventConsumeHandler.java | 9 +-
.../IgniteDiagnosticPrepareContext.java | 4 +-
.../internal/jdbc/thin/JdbcThinStatement.java | 7 +-
.../managers/deployment/GridDeploymentInfo.java | 1 +
.../deployment/GridDeploymentInfoBean.java | 21 +-
.../discovery/GridDiscoveryManager.java | 26 +-
.../pagemem/store/IgnitePageStoreManager.java | 4 +-
.../cache/CacheAffinitySharedManager.java | 4 +-
.../processors/cache/CacheMetricsImpl.java | 2 +-
.../processors/cache/GridCacheAdapter.java | 18 +-
.../cache/GridCacheDeploymentManager.java | 54 +-
.../processors/cache/GridCacheIoManager.java | 9 +-
.../processors/cache/GridCacheMapEntry.java | 9 +-
.../processors/cache/GridCacheMvccManager.java | 5 +-
.../GridDistributedTxRemoteAdapter.java | 2 +-
.../dht/GridDhtPartitionTopologyImpl.java | 13 +-
.../distributed/dht/GridDhtTxFinishFuture.java | 2 +-
.../cache/distributed/dht/GridDhtTxLocal.java | 6 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 4 +-
.../colocated/GridDhtColocatedLockFuture.java | 79 ++-
.../GridDhtPartitionsExchangeFuture.java | 2 +-
.../preloader/GridDhtPartitionsFullMessage.java | 2 +-
.../distributed/near/GridNearLockFuture.java | 116 ++--
.../near/GridNearOptimisticTxPrepareFuture.java | 3 +-
.../near/GridNearTransactionalCache.java | 3 -
.../near/GridNearTxFastFinishFuture.java | 82 +++
.../near/GridNearTxFinishFuture.java | 27 +-
.../cache/distributed/near/GridNearTxLocal.java | 309 ++++++---
.../distributed/near/NearTxFinishFuture.java | 5 +
.../GridCacheDatabaseSharedManager.java | 55 +-
.../persistence/GridCacheOffheapManager.java | 2 +-
.../IgniteCacheDatabaseSharedManager.java | 2 +-
.../cache/persistence/file/AsyncFileIO.java | 218 ++++++
.../persistence/file/AsyncFileIOFactory.java | 52 ++
.../cache/persistence/file/FileIOFactory.java | 25 +-
.../cache/persistence/file/FilePageStore.java | 7 +-
.../persistence/file/FilePageStoreManager.java | 7 +-
.../file/FileVersionCheckingFactory.java | 2 +-
.../persistence/file/RandomAccessFileIO.java | 48 +-
.../file/RandomAccessFileIOFactory.java | 14 +-
.../persistence/pagemem/PageMemoryImpl.java | 96 ++-
.../persistence/pagemem/PagesWriteThrottle.java | 104 +++
.../cache/persistence/tree/io/DataPageIO.java | 6 +-
.../cache/persistence/tree/io/IOVersions.java | 3 +
.../cache/persistence/tree/io/PageIO.java | 2 +-
.../wal/AbstractWalRecordsIterator.java | 2 +-
.../cache/persistence/wal/FileInput.java | 7 +
.../wal/FileWriteAheadLogManager.java | 8 +-
.../reader/StandaloneWalRecordsIterator.java | 4 +-
.../continuous/CacheContinuousQueryHandler.java | 9 +-
.../cache/transactions/IgniteInternalTx.java | 2 +-
.../cache/transactions/IgniteTxAdapter.java | 13 +-
.../cache/transactions/IgniteTxHandler.java | 26 +-
.../transactions/IgniteTxLocalAdapter.java | 10 +-
.../cache/transactions/IgniteTxLocalEx.java | 7 +-
.../cache/transactions/IgniteTxManager.java | 80 +--
.../odbc/odbc/OdbcConnectionContext.java | 6 +-
.../processors/odbc/odbc/OdbcMessageParser.java | 17 +-
.../odbc/odbc/OdbcQueryExecuteBatchResult.java | 16 +-
.../odbc/odbc/OdbcRequestHandler.java | 54 +-
.../processors/odbc/odbc/OdbcUtils.java | 24 +-
.../processors/query/GridQueryProcessor.java | 30 +-
.../timeout/GridTimeoutProcessor.java | 23 +-
.../internal/util/future/GridFutureAdapter.java | 16 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 3 +-
.../tcp/internal/TcpDiscoveryNode.java | 4 +-
.../resources/META-INF/classnames.properties | 2 +
.../processors/cache/CacheTxFastFinishTest.java | 9 +-
.../cache/IgniteTxConfigCacheSelfTest.java | 14 +
.../IgniteCacheThreadLocalTxTest.java | 223 +++++++
.../IgniteOptimisticTxSuspendResumeTest.java | 6 +-
...ionedMultiNodeLongTxTimeout2FullApiTest.java | 34 +
.../IgnitePdsContinuousRestartTest.java | 45 +-
...ousRestartTestWithSharedGroupAndIndexes.java | 67 ++
...ckpointSimulationWithRealCpDisabledTest.java | 4 +-
.../file/IgnitePdsThreadInterruptionTest.java | 205 ++++++
.../db/wal/IgniteWalFlushFailoverTest.java | 22 +-
.../db/wal/crc/IgniteDataIntegrityTests.java | 39 +-
.../pagemem/BPlusTreePageMemoryImplTest.java | 4 +-
.../BPlusTreeReuseListPageMemoryImplTest.java | 3 +-
.../MetadataStoragePageMemoryImplTest.java | 4 +-
.../pagemem/NoOpPageStoreManager.java | 2 +-
.../pagemem/PageMemoryImplNoLoadTest.java | 4 +-
.../persistence/pagemem/PageMemoryImplTest.java | 4 +-
.../pagemem/PagesWriteThrottleSandboxTest.java | 264 ++++++++
.../pagemem/PagesWriteThrottleSmokeTest.java | 327 +++++++++
.../TxRollbackOnTimeoutNearCacheTest.java | 28 +
...ollbackOnTimeoutNoDeadlockDetectionTest.java | 47 ++
.../transactions/TxRollbackOnTimeoutTest.java | 655 +++++++++++++++++++
.../IgniteCacheFullApiSelfTestSuite.java | 2 +
.../testsuites/IgniteCacheTestSuite6.java | 10 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 4 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
.../development/utils/IgniteWalConverter.java | 1 -
.../hadoop/impl/HadoopTxConfigCacheTest.java | 4 +-
.../processors/query/h2/IgniteH2Indexing.java | 9 +
.../processors/query/h2/opt/GridH2Table.java | 2 +
.../IgnitePersistentStoreSchemaLoadTest.java | 237 ++++---
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 +
modules/platforms/cpp/README.md | 222 +++++++
modules/platforms/cpp/odbc-test/Makefile.am | 1 +
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 1 +
.../project/vs/odbc-test.vcxproj.filters | 3 +
.../platforms/cpp/odbc-test/src/errors_test.cpp | 289 ++++++++
.../platforms/cpp/odbc-test/src/parser_test.cpp | 4 +-
.../cpp/odbc-test/src/queries_test.cpp | 16 +-
modules/platforms/cpp/odbc/include/Makefile.am | 1 +
.../cpp/odbc/include/ignite/odbc/common_types.h | 131 +++-
.../ignite/odbc/diagnostic/diagnosable.h | 9 +
.../odbc/diagnostic/diagnosable_adapter.h | 8 +
.../cpp/odbc/include/ignite/odbc/message.h | 48 +-
.../cpp/odbc/include/ignite/odbc/odbc_error.h | 94 +++
.../cpp/odbc/include/ignite/odbc/parser.h | 35 +-
.../platforms/cpp/odbc/project/vs/odbc.vcxproj | 1 +
.../cpp/odbc/project/vs/odbc.vcxproj.filters | 3 +
modules/platforms/cpp/odbc/src/common_types.cpp | 51 ++
modules/platforms/cpp/odbc/src/connection.cpp | 25 +-
.../odbc/src/diagnostic/diagnosable_adapter.cpp | 8 +
.../odbc/src/diagnostic/diagnostic_record.cpp | 105 ++-
modules/platforms/cpp/odbc/src/message.cpp | 37 +-
.../cpp/odbc/src/query/batch_query.cpp | 13 +-
.../odbc/src/query/column_metadata_query.cpp | 11 +-
.../platforms/cpp/odbc/src/query/data_query.cpp | 31 +-
.../cpp/odbc/src/query/table_metadata_query.cpp | 11 +-
modules/platforms/cpp/odbc/src/statement.cpp | 11 +-
.../Examples/Example.cs | 6 +
modules/platforms/dotnet/README.md | 237 +++++--
.../Datagrid/EntryProcessorExample.cs | 4 +
.../app/modules/agent/AgentManager.service.js | 8 +-
.../ignite/console/demo/AgentMetadataDemo.java | 7 +
136 files changed, 5152 insertions(+), 797 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index b2f2774,9c7451e..03b7b6e
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@@ -1415,11 -1415,9 +1415,11 @@@ public class GridDhtPartitionsExchangeF
cacheCtx.continuousQueries().flushBackupQueue(res);
}
- }
+ }
if (err == null) {
+ cctx.coordinators().assignCoordinator(exchCtx.events().discoveryCache());
+
if (centralizedAff) {
assert !exchCtx.mergeExchanges();
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index b1748b8,b6a8855..92f6b86
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@@ -136,13 -135,11 +136,18 @@@ public final class GridNearTxFinishFutu
}
}
+ /** {@inheritDoc} */
+ @Override public boolean commit() {
+ return commit;
+ }
+
+ /**
+ * @return Cache context.
+ */
+ GridCacheSharedContext<K, V> context() {
+ return cctx;
+ }
+
/** {@inheritDoc} */
@Override public IgniteUuid futureId() {
return futId;
@@@ -390,9 -394,14 +402,15 @@@
fut.getClass() == CheckRemoteTxMiniFuture.class;
}
+ /**
+ * Initializes future.
+ *
+ * @param commit Commit flag.
+ * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
+ */
@SuppressWarnings("ForLoopReplaceableByForEach")
+ /** {@inheritDoc} */
- public void finish(boolean commit) {
+ public void finish(boolean commit, boolean clearThreadMap) {
if (tx.onNeedCheckBackup()) {
assert tx.onePhaseCommit();
@@@ -405,28 -414,8 +423,27 @@@
return;
}
+ if (!commit && tx.mvccCoordinatorVersion() != null) {
+ ClusterNode crd = cctx.coordinators().coordinator(tx.topologyVersion());
+
+ assert crd != null;
+
+ cctx.coordinators().ackTxRollback(crd, tx.nearXidVersion());
+ }
+
try {
- if (tx.localFinish(commit) || (!commit && tx.state() == UNKNOWN)) {
+ if (tx.localFinish(commit, clearThreadMap) || (!commit && tx.state() == UNKNOWN)) {
+ GridLongList waitTxs = tx.mvccWaitTransactions();
+
+ if (waitTxs != null) {
+ ClusterNode crd = cctx.coordinators().coordinator(tx.topologyVersion());
+
+ assert crd != null;
+
+ IgniteInternalFuture fut = cctx.coordinators().waitTxsFuture(crd, waitTxs);
+
+ add(fut);
+ }
-
if ((tx.onePhaseCommit() && needFinishOnePhase(commit)) || (!tx.onePhaseCommit() && mappings != null)) {
if (mappings.single()) {
GridDistributedTxMapping mapping = mappings.singleMapping();
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
index 94224ca,132c754..9223c9f
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
@@@ -23,9 -23,9 +23,14 @@@ import org.apache.ignite.internal.proce
/**
*
*/
-public interface NearTxFinishFuture extends IgniteInternalFuture<IgniteInternalTx> {
+public interface NearTxFinishFuture extends IgniteInternalFuture<IgniteInternalTx> {
/**
+ * @return Commit flag.
+ */
+ boolean commit();
++
++ /**
+ * @param commit {@code True} to commit, otherwise rollback.
+ */
+ public void finish(boolean commit);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
index 6445304,9e06d9d..091ecc5
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
@@@ -635,9 -634,4 +635,9 @@@ public interface IgniteInternalTx
* @param e Commit error.
*/
public void commitError(Throwable e);
+
+ /**
+ * @param mvccVer Version.
+ */
+ public void mvccCoordinatorVersion(MvccCoordinatorVersion mvccVer);
- }
+ }
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index fd36fca,b5178b5..ee7dfd2
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@@ -374,15 -370,13 +374,22 @@@ public abstract class IgniteTxAdapter e
consistentIdMapper = new ConsistentIdMapper(cctx.discovery());
}
+ public MvccCoordinatorVersion mvccCoordinatorVersion() {
+ return mvccVer;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mvccCoordinatorVersion(MvccCoordinatorVersion mvccVer) {
+ this.mvccVer = mvccVer;
+ }
+
+ /**
+ * @return Shared cache context.
+ */
+ public GridCacheSharedContext<?, ?> context() {
+ return cctx;
+ }
+
/** {@inheritDoc} */
@Override public boolean localResult() {
assert originatingNodeId() != null;
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreePageMemoryImplTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/BPlusTreeReuseListPageMemoryImplTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/MetadataStoragePageMemoryImplTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplNoLoadTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/3ad260d2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImplTest.java
----------------------------------------------------------------------
[35/35] ignite git commit: Merge remote-tracking branch
'remotes/origin/ignite-3478' into ignite-3479
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/ignite-3478' into ignite-3479
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/db7534b4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/db7534b4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/db7534b4
Branch: refs/heads/ignite-3479
Commit: db7534b43d1f3db271658b70acf2ec451f39a7c9
Parents: 65203ce 0c4325a
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 14:21:18 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 14:21:18 2017 +0300
----------------------------------------------------------------------
README.md | 149 +++--
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 3 +
.../jdbc/thin/JdbcThinErrorsSelfTest.java | 31 +
.../thin/JdbcThinSelectAfterAlterTable.java | 173 +++++
.../apache/ignite/IgniteSystemProperties.java | 10 +
.../PersistentStoreConfiguration.java | 34 +-
.../internal/GridEventConsumeHandler.java | 9 +-
.../IgniteDiagnosticPrepareContext.java | 4 +-
.../internal/jdbc/thin/JdbcThinStatement.java | 7 +-
.../managers/deployment/GridDeploymentInfo.java | 1 +
.../deployment/GridDeploymentInfoBean.java | 21 +-
.../discovery/GridDiscoveryManager.java | 26 +-
.../pagemem/store/IgnitePageStoreManager.java | 4 +-
.../cache/CacheAffinitySharedManager.java | 4 +-
.../processors/cache/CacheMetricsImpl.java | 2 +-
.../processors/cache/GridCacheAdapter.java | 18 +-
.../cache/GridCacheDeploymentManager.java | 54 +-
.../processors/cache/GridCacheIoManager.java | 9 +-
.../processors/cache/GridCacheMapEntry.java | 9 +-
.../processors/cache/GridCacheMvccManager.java | 5 +-
.../GridDistributedTxRemoteAdapter.java | 2 +-
.../dht/GridDhtPartitionTopologyImpl.java | 13 +-
.../distributed/dht/GridDhtTxFinishFuture.java | 2 +-
.../cache/distributed/dht/GridDhtTxLocal.java | 6 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 4 +-
.../dht/GridPartitionedGetFuture.java | 2 +-
.../colocated/GridDhtColocatedLockFuture.java | 79 ++-
.../GridDhtPartitionsExchangeFuture.java | 2 +-
.../preloader/GridDhtPartitionsFullMessage.java | 2 +-
.../distributed/near/GridNearLockFuture.java | 116 ++--
.../near/GridNearOptimisticTxPrepareFuture.java | 3 +-
.../near/GridNearTransactionalCache.java | 3 -
.../near/GridNearTxFastFinishFuture.java | 80 +++
.../near/GridNearTxFinishAndAckFuture.java | 11 +-
.../near/GridNearTxFinishFuture.java | 26 +-
.../cache/distributed/near/GridNearTxLocal.java | 311 ++++++---
.../distributed/near/NearTxFinishFuture.java | 8 +-
.../mvcc/CacheCoordinatorsSharedManager.java | 29 +-
.../GridCacheDatabaseSharedManager.java | 55 +-
.../persistence/GridCacheOffheapManager.java | 2 +-
.../IgniteCacheDatabaseSharedManager.java | 2 +-
.../cache/persistence/file/AsyncFileIO.java | 218 ++++++
.../persistence/file/AsyncFileIOFactory.java | 52 ++
.../cache/persistence/file/FileIOFactory.java | 25 +-
.../cache/persistence/file/FilePageStore.java | 7 +-
.../persistence/file/FilePageStoreManager.java | 7 +-
.../file/FileVersionCheckingFactory.java | 2 +-
.../persistence/file/RandomAccessFileIO.java | 48 +-
.../file/RandomAccessFileIOFactory.java | 14 +-
.../persistence/pagemem/PageMemoryImpl.java | 96 ++-
.../persistence/pagemem/PagesWriteThrottle.java | 104 +++
.../cache/persistence/tree/io/DataPageIO.java | 6 +-
.../cache/persistence/tree/io/IOVersions.java | 3 +
.../cache/persistence/tree/io/PageIO.java | 2 +-
.../wal/AbstractWalRecordsIterator.java | 2 +-
.../cache/persistence/wal/FileInput.java | 7 +
.../wal/FileWriteAheadLogManager.java | 8 +-
.../reader/StandaloneWalRecordsIterator.java | 4 +-
.../query/GridCacheDistributedQueryManager.java | 2 +-
.../cache/query/GridCacheQueryManager.java | 2 +-
.../continuous/CacheContinuousQueryHandler.java | 9 +-
.../cache/transactions/IgniteInternalTx.java | 2 +-
.../cache/transactions/IgniteTxAdapter.java | 13 +-
.../cache/transactions/IgniteTxHandler.java | 26 +-
.../transactions/IgniteTxLocalAdapter.java | 10 +-
.../cache/transactions/IgniteTxLocalEx.java | 7 +-
.../cache/transactions/IgniteTxManager.java | 80 +--
.../odbc/odbc/OdbcConnectionContext.java | 6 +-
.../processors/odbc/odbc/OdbcMessageParser.java | 17 +-
.../odbc/odbc/OdbcQueryExecuteBatchResult.java | 16 +-
.../odbc/odbc/OdbcRequestHandler.java | 54 +-
.../processors/odbc/odbc/OdbcUtils.java | 24 +-
.../processors/query/GridQueryProcessor.java | 30 +-
.../timeout/GridTimeoutProcessor.java | 23 +-
.../internal/util/future/GridFutureAdapter.java | 16 +
.../ignite/spi/discovery/tcp/ServerImpl.java | 3 +-
.../tcp/internal/TcpDiscoveryNode.java | 4 +-
.../resources/META-INF/classnames.properties | 2 +
.../processors/cache/CacheTxFastFinishTest.java | 9 +-
.../cache/IgniteTxConfigCacheSelfTest.java | 14 +
.../IgniteCacheThreadLocalTxTest.java | 223 +++++++
.../IgniteOptimisticTxSuspendResumeTest.java | 6 +-
...ionedMultiNodeLongTxTimeout2FullApiTest.java | 34 +
.../cache/mvcc/CacheMvccTransactionsTest.java | 121 +++-
.../IgnitePdsContinuousRestartTest.java | 45 +-
...ousRestartTestWithSharedGroupAndIndexes.java | 67 ++
...ckpointSimulationWithRealCpDisabledTest.java | 4 +-
.../file/IgnitePdsThreadInterruptionTest.java | 205 ++++++
.../db/wal/IgniteWalFlushFailoverTest.java | 22 +-
.../db/wal/crc/IgniteDataIntegrityTests.java | 39 +-
.../pagemem/BPlusTreePageMemoryImplTest.java | 4 +-
.../BPlusTreeReuseListPageMemoryImplTest.java | 3 +-
.../MetadataStoragePageMemoryImplTest.java | 4 +-
.../pagemem/NoOpPageStoreManager.java | 2 +-
.../pagemem/PageMemoryImplNoLoadTest.java | 4 +-
.../persistence/pagemem/PageMemoryImplTest.java | 4 +-
.../pagemem/PagesWriteThrottleSandboxTest.java | 264 ++++++++
.../pagemem/PagesWriteThrottleSmokeTest.java | 327 +++++++++
.../TxRollbackOnTimeoutNearCacheTest.java | 28 +
...ollbackOnTimeoutNoDeadlockDetectionTest.java | 47 ++
.../transactions/TxRollbackOnTimeoutTest.java | 655 +++++++++++++++++++
.../IgniteCacheFullApiSelfTestSuite.java | 2 +
.../testsuites/IgniteCacheTestSuite6.java | 10 +
.../ignite/testsuites/IgnitePdsTestSuite.java | 4 +
.../ignite/testsuites/IgnitePdsTestSuite2.java | 3 +
.../development/utils/IgniteWalConverter.java | 1 -
.../hadoop/impl/HadoopTxConfigCacheTest.java | 4 +-
.../processors/query/h2/IgniteH2Indexing.java | 9 +
.../processors/query/h2/opt/GridH2Table.java | 2 +
.../IgnitePersistentStoreSchemaLoadTest.java | 237 ++++---
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 +
modules/platforms/cpp/README.md | 222 +++++++
modules/platforms/cpp/odbc-test/Makefile.am | 1 +
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 1 +
.../project/vs/odbc-test.vcxproj.filters | 3 +
.../platforms/cpp/odbc-test/src/errors_test.cpp | 289 ++++++++
.../platforms/cpp/odbc-test/src/parser_test.cpp | 4 +-
.../cpp/odbc-test/src/queries_test.cpp | 16 +-
modules/platforms/cpp/odbc/include/Makefile.am | 1 +
.../cpp/odbc/include/ignite/odbc/common_types.h | 131 +++-
.../ignite/odbc/diagnostic/diagnosable.h | 9 +
.../odbc/diagnostic/diagnosable_adapter.h | 8 +
.../cpp/odbc/include/ignite/odbc/message.h | 48 +-
.../cpp/odbc/include/ignite/odbc/odbc_error.h | 94 +++
.../cpp/odbc/include/ignite/odbc/parser.h | 35 +-
.../platforms/cpp/odbc/project/vs/odbc.vcxproj | 1 +
.../cpp/odbc/project/vs/odbc.vcxproj.filters | 3 +
modules/platforms/cpp/odbc/src/common_types.cpp | 51 ++
modules/platforms/cpp/odbc/src/connection.cpp | 25 +-
.../odbc/src/diagnostic/diagnosable_adapter.cpp | 8 +
.../odbc/src/diagnostic/diagnostic_record.cpp | 105 ++-
modules/platforms/cpp/odbc/src/message.cpp | 37 +-
.../cpp/odbc/src/query/batch_query.cpp | 13 +-
.../odbc/src/query/column_metadata_query.cpp | 11 +-
.../platforms/cpp/odbc/src/query/data_query.cpp | 31 +-
.../cpp/odbc/src/query/table_metadata_query.cpp | 11 +-
modules/platforms/cpp/odbc/src/statement.cpp | 11 +-
.../Examples/Example.cs | 6 +
modules/platforms/dotnet/README.md | 237 +++++--
.../Datagrid/EntryProcessorExample.cs | 4 +
.../app/modules/agent/AgentManager.service.js | 8 +-
.../ignite/console/demo/AgentMetadataDemo.java | 7 +
142 files changed, 5305 insertions(+), 813 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/db7534b4/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
----------------------------------------------------------------------
diff --cc modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
index f071141,d266bb3..893b1f9
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
@@@ -571,10 -532,21 +584,16 @@@ public class CacheCoordinatorsSharedMan
MvccCoordinatorVersionResponse res = new MvccCoordinatorVersionResponse();
- for (Long txVer : activeTxs.values())
+ Long trackCntr = mvccCntr;
+
+ for (Long txVer : activeTxs.values()) {
+ if (txVer < trackCntr)
+ trackCntr = txVer;
+
res.addTx(txVer);
+ }
- Integer queries = activeQueries.get(trackCntr);
-
- if (queries != null)
- activeQueries.put(trackCntr, queries + 1);
- else
- activeQueries.put(trackCntr, 1);
+ TestDebugLog.addEntryMessage(mvccCntr, res.activeTransactions(), "qry cntr");
res.init(futId, crdVer, mvccCntr, COUNTER_NA);
http://git-wip-us.apache.org/repos/asf/ignite/blob/db7534b4/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
----------------------------------------------------------------------
[22/35] ignite git commit: ignite-3479
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java b/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java
deleted file mode 100644
index f008c32..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java
+++ /dev/null
@@ -1,1713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.GridAtomicLong;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.NotNull;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- *
- */
-public class MvccTestApp3 {
- /** */
- private static final boolean DEBUG_LOG = false;
-
- /** */
- private static final boolean SQL = false;
-
- public static void main1(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 2, true);
-
- Map<Object, Object> vals = cluster.sqlAll();
-
- System.out.println();
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)getData.get(i);
-
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main0(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txRemoveTransfer(0, 1);
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (Map.Entry<Object, Object> e : getData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main(String[] args) throws Exception {
- final AtomicBoolean err = new AtomicBoolean();
-
- final int READ_THREADS = 4;
- final int UPDATE_THREADS = 4;
- final int ACCOUNTS = 50;
-
- final int START_VAL = 100000;
-
- for (int iter = 0; iter < 1000; iter++) {
- System.out.println("Iteration [readThreads=" + READ_THREADS +
- ", updateThreads=" + UPDATE_THREADS + ", accounts=" + ACCOUNTS + ", iter=" + iter + ']');
-
- final TestCluster cluster = new TestCluster(1);
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- List<Thread> threads = new ArrayList<>();
-
- Thread cleanupThread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("cleanup");
-
- try {
- while (!stop.get()) {
- cluster.cleanup();
-
- Thread.sleep(1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- threads.add(cleanupThread);
-
- cleanupThread.start();
-
- final boolean REMOVES = false;
-
- for (int i = 0; i < READ_THREADS; i++) {
- final int id = i;
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("read" + id);
-
- int cnt = 0;
-
- while (!stop.get()) {
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- cnt++;
-
- int sum = 0;
-
- if (REMOVES) {
- for (Map.Entry<Object, Object> e : qryData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
- else
- System.out.println("With null");
- }
- }
- else {
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- if (val == null) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("No value for key: " + i);
- }
-
- return;
- }
-
- sum += val;
- }
- }
-
- if (sum != ACCOUNTS * START_VAL) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("Invalid get sum: " + sum);
- }
- }
- }
-
- System.out.println("Get cnt: " + cnt);
- }
- catch (Throwable e) {
- e.printStackTrace();
-
- err.set(true);
- stop.set(true);
- }
- }
- });
-
- threads.add(thread);
-
- thread.start();
- }
-
- for (int i = 0; i < UPDATE_THREADS; i++) {
- final int id = i;
-
- Thread thread;
-
- if (REMOVES) {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (rnd.nextBoolean()) {
- cluster.txRemoveTransfer(id1, id2);
- }
- else
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
- else {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (id1 > id2) {
- int tmp = id1;
- id1 = id2;
- id2 = tmp;
- }
-
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
-
- threads.add(thread);
-
- thread.start();
- }
-
- long endTime = System.currentTimeMillis() + 2_000;
-
- while (!stop.get()) {
- Thread.sleep(1000);
-
- if (System.currentTimeMillis() >= endTime)
- break;
-
- //cluster.dumpMvccInfo();
- }
-
- stop.set(true);
-
- for (Thread thread : threads)
- thread.join();
-
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- System.out.println("Val " + val);
-
- if (val != null)
- sum += val;
- }
-
- System.out.println("Sum=" + sum + ", expSum=" + (ACCOUNTS * START_VAL));
-
- if (err.get()) {
- System.out.println("Error!");
-
- System.exit(1);
- }
-
-// cluster.dumpMvccInfo();
-//
-// System.out.println("Cleanup");
-//
-// cluster.cleanup();
-//
-// cluster.dumpMvccInfo();
-
- TestDebugLog.clear();
- }
- }
-
- /**
- *
- */
- static class TestCluster {
- /** */
- final List<Node> nodes = new ArrayList<>();
-
- /** */
- final Coordinator crd;
-
- /** */
- final AtomicLong txIdGen = new AtomicLong(10_000);
-
- TestCluster(int nodesNum) {
- crd = new Coordinator();
-
- for (int i = 0; i < nodesNum; i++)
- nodes.add(new Node(i));
- }
-
- void cleanup() {
- CoordinatorCounter cntr = crd.cleanupVersion();
-
- for (Node node : nodes)
- node.dataStore.cleanup(cntr);
- }
-
- void txPutAll(Map<Object, Object> data) {
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- for (Object key : data.keySet()) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), data.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, cntr.cntr);
- }
-
- void txTransfer(Integer id1, Integer id2, boolean fromFirst) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(id1);
- keys.add(id2);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer curVal1 = (Integer)vals.get(id1);
- Integer curVal2 = (Integer)vals.get(id2);
-
- boolean update = false;
-
- Integer newVal1 = null;
- Integer newVal2 = null;
-
- if (curVal1 != null && curVal2 != null) {
- if (fromFirst) {
- if (curVal1 > 0) {
- update = true;
-
- newVal1 = curVal1 - 1;
- newVal2 = curVal2 + 1;
- }
- }
- else {
- if (curVal2 > 0) {
- update = true;
-
- newVal1 = curVal1 + 1;
- newVal2 = curVal2 - 1;
- }
- }
- }
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(id1, newVal1);
- newVals.put(id2, newVal2);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("update", txId, id1, newVal1, id2, newVal2, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId, cntr.cntr);
-
-// if (DEBUG_LOG)
-// TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- void txRemoveTransfer(Integer from, Integer to) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(from);
- keys.add(to);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer fromVal = (Integer)vals.get(from);
- Integer toVal = (Integer)vals.get(to);
-
- boolean update = fromVal != null && toVal != null;
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(from, null);
- newVals.put(to, fromVal + toVal);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("remove", txId, from, fromVal, to, toVal, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId, cntr.cntr);
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- public void dumpMvccInfo() {
- for (Node node : nodes) {
- int sql = node.dataStore.mvccSqlIdx.size();
-
- for (Map.Entry<Object, MvccValue> e : node.dataStore.mainIdx.entrySet()) {
- List<MvccValue> list = node.dataStore.mvccIdx.get(e.getKey());
-
- int size = 0;
-
- if (list != null) {
- synchronized (list) {
- size = list.size();
- }
- }
-
- System.out.println("Mvcc info [key=" + e.getKey() +
- ", val=" + e.getValue() +
- ", mvccVals=" + size +
- ", sqlVals=" + sql + ']');
- }
- }
- }
-
- public Map<Object, Object> sqlAll() {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Node node : nodes) {
- Map<Object, Object> nodeRes = node.dataStore.sqlQuery(qryVer);
-
- res.putAll(nodeRes);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sqlAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- public Map<Object, Object> getAll(Set<?> keys) {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Object val = node.dataStore.get(key, qryVer);
-
- res.put(key, val);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("getAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- private int nodeForKey(Object key) {
- return U.safeAbs(key.hashCode()) % nodes.size();
- }
- }
-
- /**
- *
- */
- static class Node {
- /** */
- final DataStore dataStore;
-
- /** */
- final int nodexIdx;
-
- public Node(int nodexIdx) {
- this.nodexIdx = nodexIdx;
-
- dataStore = new DataStore();
- }
-
- @Override public String toString() {
- return "Node [idx=" + nodexIdx + ']';
- }
- }
-
- /**
- *
- */
- static class Coordinator {
- /** */
- private final AtomicLong cntr = new AtomicLong(-1);
-
- /** */
- private final GridAtomicLong commitCntr = new GridAtomicLong(-1);
-
- /** */
- private final ConcurrentHashMap8<Long, QueryCounter> activeQueries = new ConcurrentHashMap8<>();
-
- /** */
- @GridToStringInclude
- private final ConcurrentHashMap8<TxId, Long> activeTxs = new ConcurrentHashMap8<>();
-
- CoordinatorCounter nextTxCounter(TxId txId) {
- long cur = cntr.get();
-
- activeTxs.put(txId, cur + 1);
-
- CoordinatorCounter newCtr = new CoordinatorCounter(cntr.incrementAndGet());
-
- return newCtr;
- }
-
- void txDone(TxId txId, long cntr) {
- Long rmvd = activeTxs.remove(txId);
-
- assert rmvd != null;
-
- commitCntr.setIfGreater(cntr);
- }
-
- private GridAtomicLong minActive0 = new GridAtomicLong(0);
-
- private Long minActive(Set<TxId> txs) {
- Long minActive = null;
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet()) {
- if (txs != null)
- txs.add(e.getKey());
-
-// TxId val = e.getValue();
-//
-// while (val.cntr == -1)
-// Thread.yield();
-
- long cntr = e.getValue();
-
- if (minActive == null)
- minActive = cntr;
- else if (cntr < minActive)
- minActive = cntr;
- }
-
- if (minActive != null) {
- if (!minActive0.setIfGreater(minActive))
- return minActive0.get();
- }
-
- return minActive;
- }
-
- static class QueryCounter extends AtomicInteger {
- public QueryCounter(int initialValue) {
- super(initialValue);
- }
-
- boolean increment2() {
- for (;;) {
- int current = get();
- int next = current + 1;
-
- if (current == 0)
- return false;
-
- if (compareAndSet(current, next))
- return true;
- }
- }
- }
-
- private ReadWriteLock rwLock = new ReentrantReadWriteLock();
-
- MvccQueryVersion queryVersion() {
- rwLock.readLock().lock();
-
- long useCntr = commitCntr.get();
-
- Set<TxId> txs = new HashSet<>();
-
- Long minActive = minActive(txs);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- MvccQueryVersion qryVer = new MvccQueryVersion(new CoordinatorCounter(useCntr), txs);
-
- for (;;) {
- QueryCounter qryCnt = activeQueries.get(useCntr);
-
- if (qryCnt != null) {
- boolean inc = qryCnt.increment2();
-
- if (!inc) {
- activeQueries.remove(useCntr, qryCnt);
-
- continue;
- }
- }
- else {
- qryCnt = new QueryCounter(1);
-
- if (activeQueries.putIfAbsent(useCntr, qryCnt) != null)
- continue;
- }
-
- break;
- }
-
- rwLock.readLock().unlock();
-
- return qryVer;
- }
-
- void queryDone(CoordinatorCounter cntr) {
- AtomicInteger qryCnt = activeQueries.get(cntr.cntr);
-
- assert qryCnt != null : cntr.cntr;
-
- int left = qryCnt.decrementAndGet();
-
- assert left >= 0 : left;
-
- if (left == 0)
- activeQueries.remove(cntr.cntr, qryCnt);
- }
-
- CoordinatorCounter cleanupVersion() {
- rwLock.writeLock().lock();
-
- long useCntr = commitCntr.get();
-
- Long minActive = minActive(null);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- for (Long qryCntr : activeQueries.keySet()) {
- if (qryCntr <= useCntr)
- useCntr = qryCntr - 1;
- }
-
- rwLock.writeLock().unlock();
-
- return new CoordinatorCounter(useCntr);
- }
-
- @Override public String toString() {
- return S.toString(Coordinator.class, this);
- }
- }
-
- /**
- *
- */
- static class CoordinatorCounter implements Comparable<CoordinatorCounter> {
- /** */
- private final long topVer; // TODO
-
- /** */
- private final long cntr;
-
- CoordinatorCounter(long cntr) {
- this.topVer = 1;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(CoordinatorCounter o) {
- return Long.compare(cntr, o.cntr);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- CoordinatorCounter that = (CoordinatorCounter)o;
-
- return cntr == that.cntr;
- }
-
- @Override public int hashCode() {
- return (int)(cntr ^ (cntr >>> 32));
- }
-
- @Override public String toString() {
- return "Cntr [c=" + cntr + ']';
- }
- }
-
- /**
- *
- */
- static class MvccUpdateVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final TxId txId;
-
- /**
- * @param cntr
- */
- MvccUpdateVersion(CoordinatorCounter cntr, TxId txId) {
- assert cntr != null;
-
- this.cntr = cntr;
- this.txId = txId;
- }
-
- @Override public String toString() {
- return S.toString(MvccUpdateVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccQueryVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- MvccQueryVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
-
- @Override public String toString() {
- return S.toString(MvccQueryVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class TxId {
- /** */
- @GridToStringInclude
- final long id;
-
- TxId(long id) {
- this.id = id;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TxId txId = (TxId) o;
-
- return id == txId.id;
- }
-
- @Override public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override public String toString() {
- return S.toString(TxId.class, this);
- }
- }
-
- /**
- *
- */
- static class SqlKey implements Comparable<SqlKey> {
- /** */
- final Comparable key;
-
- /** */
- final Comparable val;
-
- /** */
- final CoordinatorCounter cntr;
-
- public SqlKey(Object key, Object val, CoordinatorCounter cntr) {
- this.key = (Comparable)key;
- this.val = (Comparable)val;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(@NotNull SqlKey o) {
- int cmp;
-
- if (val != null && o.val != null)
- cmp = val.compareTo(o.val);
- else {
- if (val != null)
- cmp = 1;
- else
- cmp = o.val == null ? 0 : -1;
- }
-
-
- if (cmp == 0) {
- cmp = key.compareTo(o.key);
-
- if (cmp == 0)
- cmp = cntr.compareTo(o.cntr);
- }
-
- return cmp;
- }
-
- @Override public String toString() {
- return "SqlKey [key=" + key + ", val=" + val + ']';
- }
- }
-
- /**
- *
- */
- static class DataStore {
- /** */
- private final ConcurrentHashMap<Object, ReentrantLock> locks = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, MvccValue> mainIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, List<MvccValue>> mvccIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentSkipListMap<SqlKey, MvccSqlValue> mvccSqlIdx = new ConcurrentSkipListMap<>();
-
- void cleanup(CoordinatorCounter cleanupCntr) {
- for (Map.Entry<Object, List<MvccValue>> e : mvccIdx.entrySet()) {
- lockEntry(e.getKey());
-
- try {
- List<MvccValue> list = e.getValue();
-
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (val.ver.cntr.compareTo(cleanupCntr) <= 0) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6_1("cleanup",
- e.getKey(), val.val, val.ver, cleanupCntr.cntr, null, null));
- }
-
- MvccValue prev;
-
- if (val.val != null)
- prev = mainIdx.put(e.getKey(), val);
- else
- prev = mainIdx.remove(e.getKey());
-
- if (prev != null) {
- SqlKey key = new SqlKey(e.getKey(), prev.val, prev.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
-
- for (int j = 0; j <= i; j++) {
- MvccValue rmvd = list.remove(0);
-
- assert rmvd != null;
-
- if (j != i || rmvd.val == null) {
- SqlKey key = new SqlKey(e.getKey(), rmvd.val, rmvd.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
- }
-
- if (list.isEmpty())
- mvccIdx.remove(e.getKey());
-
- break;
- }
- }
- }
- }
- finally {
- unlockEntry(e.getKey());
- }
- }
- }
-
- void lockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.lock();
- }
-
- void unlockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.unlock();
- }
-
- void updateEntry(Object key, Object val, MvccUpdateVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list == null) {
- Object old = mvccIdx.putIfAbsent(key, list = new ArrayList<>());
-
- assert old == null;
- }
-
- MvccValue prevVal = null;
-
- synchronized (list) {
- if (!list.isEmpty())
- prevVal = list.get(list.size() - 1);
-
- list.add(new MvccValue(val, ver));
- }
-
- if (prevVal == null)
- prevVal = mainIdx.get(key);
-
- if (prevVal != null) {
- SqlKey prevKey = new SqlKey(key, prevVal.val, prevVal.ver.cntr);
-
- MvccSqlValue old =
- mvccSqlIdx.put(prevKey, new MvccSqlValue(prevVal.val, prevVal.ver, ver));
-
- assert old != null;
- }
-
- mvccSqlIdx.put(new SqlKey(key, val, ver.cntr), new MvccSqlValue(val, ver, null));
- }
-
- Object lastValue(Object key) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- if (list.size() > 0)
- return list.get(list.size() - 1).val;
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- return val != null ? val.val : null;
- }
-
- Map<Object, Object> sqlQuery(MvccQueryVersion qryVer) {
- Map<Object, Object> res = new HashMap<>();
-
- for (Map.Entry<SqlKey, MvccSqlValue> e : mvccSqlIdx.entrySet()) {
- MvccSqlValue val = e.getValue();
-
- if (!versionVisible(val.ver, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sql skip mvcc val", e.getKey().key, val.val, val.ver));
- }
-
- continue;
- }
-
- MvccUpdateVersion newVer = val.newVer;
-
- if (newVer != null && versionVisible(newVer, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql skip mvcc val2", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- continue;
- }
-
- Object old = res.put(e.getKey().key, e.getValue().val);
-
- if (DEBUG_LOG) {
- //TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql get mvcc val", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- if (old != null) {
- TestDebugLog.printAllAndExit("Already has value for key [key=" + e.getKey().key +
- ", qryVer=" + qryVer +
- ", oldVal=" + old +
- ", newVal=" + e.getValue().val +
- ']');
- }
-
- assert old == null;
- }
-
- return res;
- }
-
- private boolean versionVisible(MvccUpdateVersion ver, MvccQueryVersion qryVer) {
- int cmp = ver.cntr.compareTo(qryVer.cntr);
-
- return cmp <= 0;// && !qryVer.activeTxs.contains(ver.txId);
- }
-
- Object get(Object key, MvccQueryVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (!versionVisible(val.ver, ver))
- continue;
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read mvcc val", key, val.val, val.ver));
- }
-
- return val.val;
- }
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- if (val != null) {
- int cmp = val.ver.cntr.compareTo(ver.cntr);
-
- if (DEBUG_LOG) {
- if (cmp > 0) {
- synchronized (TestDebugLog.msgs) {
- TestDebugLog.msgs.add(new TestDebugLog.Message("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver.cntr + ']'));
-
- TestDebugLog.printAllAndExit("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver + ']');
- }
- }
- }
-
- assert cmp <= 0 : "Committed [ver=" + val.ver + ", qryVer=" + ver.cntr + ']';
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted val", key, val, val.ver));
- }
- else {
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted null", key, null, null));
- }
-
- return val != null ? val.val : null;
- }
-
- private ReentrantLock lock(Object key) {
- ReentrantLock e = locks.get(key);
-
- if (e == null) {
- ReentrantLock old = locks.putIfAbsent(key, e = new ReentrantLock());
-
- if (old != null)
- e = old;
- }
-
- return e;
- }
- }
-
- /**
- *
- */
- static class MvccValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- MvccValue(Object val, MvccUpdateVersion ver) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- }
-
- @Override public String toString() {
- return S.toString(MvccValue.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccSqlValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion newVer;
-
- MvccSqlValue(Object val, MvccUpdateVersion ver, MvccUpdateVersion newVer) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- this.newVer = newVer;
- }
-
- @Override public String toString() {
- return S.toString(MvccSqlValue.class, this);
- }
- }
-
- static void log(String msg) {
- System.out.println(Thread.currentThread() + ": " + msg);
- }
-
- static class TestDebugLog {
- /** */
- //static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(1_000_000));
- static final ConcurrentLinkedQueue<Object> msgs = new ConcurrentLinkedQueue<>();
-
-
-
- /** */
- private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
-
- static class Message {
- String thread = Thread.currentThread().getName();
-
- String msg;
-
- long ts = U.currentTimeMillis();
-
- public Message(String msg) {
- this.msg = msg;
- }
-
- public String toString() {
- return "Msg [msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg2 extends Message{
- Object v1;
- Object v2;
-
- public Msg2(String msg, Object v1, Object v2) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", msg=" + msg +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg3 extends Message{
- Object v1;
- Object v2;
- Object v3;
-
- public Msg3(String msg, Object v1, Object v2, Object v3) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg4 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
-
- public Msg4(String msg, Object v1, Object v2, Object v3, Object v4) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", v4=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg6 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", txId=" + v1 +
- ", id1=" + v2 +
- ", v1=" + v3 +
- ", id2=" + v4 +
- ", v2=" + v5 +
- ", cntr=" + v6 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
- static class Msg6_1 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6_1(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", key=" + v1 +
- ", val=" + v2 +
- ", ver=" + v3 +
- ", cleanupC=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class EntryMessage extends Message {
- Object key;
- Object val;
-
- public EntryMessage(Object key, Object val, String msg) {
- super(msg);
-
- this.key = key;
- this.val = val;
- }
-
- public String toString() {
- return "EntryMsg [key=" + key + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class PartMessage extends Message {
- int p;
- Object val;
-
- public PartMessage(int p, Object val, String msg) {
- super(msg);
-
- this.p = p;
- this.val = val;
- }
-
- public String toString() {
- return "PartMessage [p=" + p + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static final boolean out = false;
-
- public static void addMessage(String msg) {
- msgs.add(new Message(msg));
-
- if (out)
- System.out.println(msg);
- }
-
- public static void addEntryMessage(Object key, Object val, String msg) {
- if (key instanceof KeyCacheObject)
- key = ((KeyCacheObject)key).value(null, false);
-
- EntryMessage msg0 = new EntryMessage(key, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- public static void addPartMessage(int p, Object val, String msg) {
- PartMessage msg0 = new PartMessage(p, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- static void printAllAndExit(String msg) {
- System.out.println(msg);
-
- TestDebugLog.addMessage(msg);
-
- List<Object> msgs = TestDebugLog.printMessages(true, null);
-
- TestDebugLog.printMessages0(msgs, "test_debug_update.txt");
-
- TestDebugLog.printMessagesForThread(msgs, Thread.currentThread().getName(), "test_debug_thread.txt");
-
- System.exit(1);
- }
-
- public static void printMessagesForThread(List<Object> msgs0, String thread0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String thread = ((Message) msg).thread;
-
- if (thread.equals(thread0))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static void printMessages0(List<Object> msgs0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String msg0 = ((Message) msg).msg;
-
- if (msg0.equals("tx done") || msg0.equals("update") || msg0.equals("cleanup"))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static List<Object> printMessages(boolean file, Integer part) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (part != null && msg instanceof PartMessage) {
- if (((PartMessage) msg).p != part)
- continue;
- }
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0)
- System.out.println(msg);
- }
-
- return msgs0;
- }
-
- public static void printKeyMessages(boolean file, Object key) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- System.out.println(msg);
- }
- }
- }
-
- public static void clear() {
- msgs.clear();
- }
-
- public static void clearEntries() {
- for (Iterator it = msgs.iterator(); it.hasNext();) {
- Object msg = it.next();
-
- if (msg instanceof EntryMessage)
- it.remove();
- }
- }
-
- }}
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 78f8913..8865502 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -39,6 +39,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtInvali
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtLocalPartition;
import org.apache.ignite.internal.processors.cache.mvcc.MvccCoordinatorVersion;
import org.apache.ignite.internal.processors.cache.mvcc.MvccLongList;
+import org.apache.ignite.internal.processors.cache.mvcc.TestDebugLog;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.persistence.CacheDataRowAdapter;
import org.apache.ignite.internal.processors.cache.persistence.CacheSearchRow;
@@ -1677,6 +1678,8 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
else
visible = true;
+ TestDebugLog.addMessage3("found row", key.value(cctx.cacheObjectContext(), false), row0.mvccCounter(), visible);
+
if (visible) {
row = row0;
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
index 0f7e71e..f071141 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
@@ -23,6 +23,7 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
@@ -49,6 +50,7 @@ import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
import org.jetbrains.annotations.Nullable;
+import org.jsr166.ConcurrentHashMap8;
import org.jsr166.LongAdder8;
import static org.apache.ignite.events.EventType.EVT_NODE_FAILED;
@@ -85,7 +87,7 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
private final ConcurrentHashMap<GridCacheVersion, Long> activeTxs = new ConcurrentHashMap<>();
/** */
- private final Map<Long, Integer> activeQueries = new HashMap<>();
+ private final ConcurrentHashMap8<Long, AtomicInteger> activeQueries = new ConcurrentHashMap8<>();
/** */
private final ConcurrentMap<Long, MvccVersionFuture> verFuts = new ConcurrentHashMap<>();
@@ -483,6 +485,11 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
cleanupVer = qryVer - 1;
}
+ if (committedCntr.get() - 1 < cleanupVer)
+ cleanupVer = committedCntr.get() - 1;
+
+ TestDebugLog.addMessage3("assign tx cntr", nextCtr, res.activeTransactions(), cleanupVer);
+
res.init(futId, crdVer, nextCtr, cleanupVer);
return res;
@@ -502,32 +509,72 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
committedCntr.setIfGreater(cntr);
fut = waitTxFuts.remove(cntr);
+
+ TestDebugLog.addMessage3("tx done", cntr, null, null);
}
if (fut != null)
fut.onDone();
}
+ static boolean increment(AtomicInteger cntr) {
+ for (;;) {
+ int current = cntr.get();
+
+ if (current == 0)
+ return false;
+
+ if (cntr.compareAndSet(current, current + 1))
+ return true;
+ }
+ }
+
/**
* @param qryNodeId Node initiated query.
* @return Counter for query.
*/
- private synchronized MvccCoordinatorVersionResponse assignQueryCounter(UUID qryNodeId, long futId) {
+ private MvccCoordinatorVersionResponse assignQueryCounter(UUID qryNodeId, long futId) {
assert crdVer != 0;
- Long mvccCntr = committedCntr.get();
+ Long mvccCntr;
+
+ for(;;) {
+ mvccCntr = committedCntr.get();
+
+ for (;;) {
+ AtomicInteger qryCnt = activeQueries.get(mvccCntr);
+
+ if (qryCnt != null) {
+ boolean inc = increment(qryCnt);
+
+ if (!inc) {
+ activeQueries.remove(mvccCntr, qryCnt);
+
+ continue;
+ }
+ }
+ else {
+ qryCnt = new AtomicInteger(1);
+
+ if (activeQueries.putIfAbsent(mvccCntr, qryCnt) != null)
+ continue;
+ }
+
+ break;
+ }
+
+ if (mvccCntr == committedCntr.get())
+ break;
+
+ onQueryDone(mvccCntr);
+ }
MvccCoordinatorVersionResponse res = new MvccCoordinatorVersionResponse();
for (Long txVer : activeTxs.values())
res.addTx(txVer);
- Integer queries = activeQueries.get(mvccCntr);
-
- if (queries != null)
- activeQueries.put(mvccCntr, queries + 1);
- else
- activeQueries.put(mvccCntr, 1);
+ TestDebugLog.addEntryMessage(mvccCntr, res.activeTransactions(), "qry cntr");
res.init(futId, crdVer, mvccCntr, COUNTER_NA);
@@ -537,20 +584,19 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
/**
* @param mvccCntr Query counter.
*/
- private synchronized void onQueryDone(long mvccCntr) {
- Integer queries = activeQueries.get(mvccCntr);
+ private void onQueryDone(long mvccCntr) {
+ TestDebugLog.addEntryMessage(mvccCntr, null, "qry done");
- assert queries != null : mvccCntr;
+ AtomicInteger cntr = activeQueries.get(mvccCntr);
- int left = queries - 1;
+ assert cntr != null : mvccCntr;
- assert left >= 0 : left;
+ int left = cntr.decrementAndGet();
- if (left == 0) {
- Integer rmvd = activeQueries.remove(mvccCntr);
+ assert left >= 0 : left;
- assert rmvd != null;
- }
+ if (left == 0)
+ activeQueries.remove(mvccCntr, cntr);
}
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCoordinatorVersionResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCoordinatorVersionResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCoordinatorVersionResponse.java
index 9d61a6d..1fc24f1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCoordinatorVersionResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccCoordinatorVersionResponse.java
@@ -254,6 +254,6 @@ public class MvccCoordinatorVersionResponse implements MvccCoordinatorMessage, M
/** {@inheritDoc} */
@Override public String toString() {
- return S.toString(MvccCoordinatorVersionResponse.class, this);
+ return Arrays.toString(txs);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
index c50d63c..0164ca1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
@@ -1046,13 +1046,13 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
)
throws Exception
{
- final int ACCOUNTS = 20;
+ final int ACCOUNTS = 5;
final int ACCOUNT_START_VAL = 1000;
- final int writers = 4;
+ final int writers = 2;
- final int readers = 4;
+ final int readers = 1;
final IgniteInClosure<IgniteCache<Object, Object>> init = new IgniteInClosure<IgniteCache<Object, Object>>() {
@Override public void apply(IgniteCache<Object, Object> cache) {
@@ -1107,6 +1107,8 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
a1 = accounts.get(id1);
a2 = accounts.get(id2);
+ TestDebugLog.addMessage3("update", id1, id2, null);
+
assertNotNull(a1);
assertNotNull(a2);
@@ -1119,16 +1121,16 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
tx.commit();
}
- Map<Integer, MvccTestAccount> accounts = cache.getAll(keys);
-
- MvccTestAccount a1 = accounts.get(id1);
- MvccTestAccount a2 = accounts.get(id2);
-
- assertNotNull(a1);
- assertNotNull(a2);
-
- assertTrue(a1.updateCnt >= cntr1);
- assertTrue(a2.updateCnt >= cntr2);
+// Map<Integer, MvccTestAccount> accounts = cache.getAll(keys);
+//
+// MvccTestAccount a1 = accounts.get(id1);
+// MvccTestAccount a2 = accounts.get(id2);
+//
+// assertNotNull(a1);
+// assertNotNull(a2);
+//
+// assertTrue(a1.updateCnt >= cntr1);
+// assertTrue(a2.updateCnt >= cntr2);
}
info("Writer finished, updates: " + cnt);
@@ -1164,6 +1166,14 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
else
accounts = cache.getAll(keys);
+ if (ACCOUNTS != accounts.size()) {
+ TestDebugLog.addMessage("wrong accounts number: " + accounts.keySet());
+
+ TestDebugLog.printMessages(true, null);
+
+ System.exit(1);
+ }
+
assertEquals(ACCOUNTS, accounts.size());
int sum = 0;
@@ -1212,7 +1222,7 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
cacheParts,
writers,
readers,
- DFLT_TEST_TIME,
+ 5000,
init,
writer,
reader);
[03/35] ignite git commit: IGNITE-6448: Fixed a bug causing stale SQL
statement to reside in cache after ALTER TABLE command. This closes #2702.
Posted by sb...@apache.org.
IGNITE-6448: Fixed a bug causing stale SQL statement to reside in cache after ALTER TABLE command. This closes #2702.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c7d591ff
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c7d591ff
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c7d591ff
Branch: refs/heads/ignite-3479
Commit: c7d591ff7dbd75fb774f931816ea947dbeffc1f6
Parents: abe4ee8
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Thu Sep 21 17:46:47 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Thu Sep 21 17:46:47 2017 +0300
----------------------------------------------------------------------
.../jdbc/suite/IgniteJdbcDriverTestSuite.java | 3 +
.../thin/JdbcThinSelectAfterAlterTable.java | 173 +++++++++++++++++++
.../processors/query/h2/IgniteH2Indexing.java | 9 +
.../processors/query/h2/opt/GridH2Table.java | 2 +
4 files changed, 187 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7d591ff/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
index 3df8e2c..1ae2427 100644
--- a/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/suite/IgniteJdbcDriverTestSuite.java
@@ -55,6 +55,7 @@ import org.apache.ignite.jdbc.thin.JdbcThinNoDefaultSchemaTest;
import org.apache.ignite.jdbc.thin.JdbcThinPreparedStatementSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinResultSetSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinSchemaCaseTest;
+import org.apache.ignite.jdbc.thin.JdbcThinSelectAfterAlterTable;
import org.apache.ignite.jdbc.thin.JdbcThinStatementSelfTest;
import org.apache.ignite.jdbc.thin.JdbcThinUpdateStatementSelfTest;
@@ -149,6 +150,8 @@ public class IgniteJdbcDriverTestSuite extends TestSuite {
// New thin JDBC driver, full SQL tests
suite.addTest(new TestSuite(JdbcThinComplexDmlDdlSelfTest.class));
+ suite.addTest(new TestSuite(JdbcThinSelectAfterAlterTable.class));
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7d591ff/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinSelectAfterAlterTable.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinSelectAfterAlterTable.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinSelectAfterAlterTable.java
new file mode 100644
index 0000000..13b8f3e
--- /dev/null
+++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinSelectAfterAlterTable.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.jdbc.thin;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.ClientConnectorConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jetbrains.annotations.NotNull;
+
+/**
+ * Base class for complex SQL tests based on JDBC driver.
+ */
+public class JdbcThinSelectAfterAlterTable extends GridCommonAbstractTest {
+ /** IP finder. */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** Client connection port. */
+ private int cliPort = ClientConnectorConfiguration.DFLT_PORT;
+
+ /** JDBC connection. */
+ private Connection conn;
+
+ /** JDBC statement. */
+ private Statement stmt;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.setCacheConfiguration(cacheConfiguration(DEFAULT_CACHE_NAME));
+
+ TcpDiscoverySpi disco = new TcpDiscoverySpi();
+
+ disco.setIpFinder(IP_FINDER);
+
+ cfg.setDiscoverySpi(disco);
+
+ cfg.setClientConnectorConfiguration(new ClientConnectorConfiguration().setPort(cliPort++));
+
+ return cfg;
+ }
+
+ /**
+ * @param name Cache name.
+ * @return Cache configuration.
+ * @throws Exception In case of error.
+ */
+ private CacheConfiguration cacheConfiguration(@NotNull String name) throws Exception {
+ CacheConfiguration cfg = defaultCacheConfiguration();
+
+ cfg.setName(name);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ startGridsMultiThreaded(2);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ conn = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1");
+
+ stmt = conn.createStatement();
+
+ stmt.executeUpdate("CREATE TABLE person (id LONG, name VARCHAR, city_id LONG, PRIMARY KEY (id, city_id))");
+ stmt.executeUpdate("INSERT INTO person (id, name, city_id) values (1, 'name_1', 11)");
+
+ stmt.executeQuery("select * from person");
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stmt.close();
+ conn.close();
+
+ // Destroy all SQL caches after test.
+ for (String cacheName : grid(0).cacheNames()) {
+ DynamicCacheDescriptor cacheDesc = grid(0).context().cache().cacheDescriptor(cacheName);
+
+ if (cacheDesc != null && cacheDesc.sql())
+ grid(0).destroyCache0(cacheName, true);
+ }
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "unchecked"})
+ public void testSelectAfterAlterTableSingleNode() throws Exception {
+ stmt.executeUpdate("alter table person add age int");
+
+ checkNewColumn(stmt);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings({"ThrowableResultOfMethodCallIgnored", "unchecked"})
+ public void testSelectAfterAlterTableMultiNode() throws Exception {
+ try (Connection conn2 = DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:"
+ + (ClientConnectorConfiguration.DFLT_PORT + 1))) {
+
+ try (Statement stmt2 = conn2.createStatement()) {
+ stmt2.executeUpdate("alter table person add age int");
+ }
+ }
+
+ checkNewColumn(stmt);
+ }
+
+ /**
+ * @param stmt Statement to check new column.
+ * @throws SQLException If failed.
+ */
+ public void checkNewColumn(Statement stmt) throws SQLException {
+ ResultSet rs = stmt.executeQuery("select * from person");
+
+ ResultSetMetaData meta = rs.getMetaData();
+
+ assertEquals(4, meta.getColumnCount());
+
+ boolean newColExists = false;
+
+ for (int i = 1; i <= meta.getColumnCount(); ++i) {
+ if ("age".equalsIgnoreCase(meta.getColumnName(i))) {
+ newColExists = true;
+
+ break;
+ }
+ }
+
+ assertTrue(newColExists);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7d591ff/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
index a07d48e..03e2391 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/IgniteH2Indexing.java
@@ -729,6 +729,8 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
desc.table().addColumns(cols, ifColNotExists);
+
+ clearCachedQueries();
}
/**
@@ -2244,6 +2246,13 @@ public class IgniteH2Indexing implements GridQueryIndexing {
}
}
+ /**
+ * Remove all cached queries from cached two-steps queries.
+ */
+ public void clearCachedQueries() {
+ twoStepCache.clear();
+ }
+
/** {@inheritDoc} */
@Override public IndexingQueryFilter backupFilter(@Nullable final AffinityTopologyVersion topVer,
@Nullable final int[] parts) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7d591ff/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
index 45bb9c7..549cbfb 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2Table.java
@@ -999,6 +999,8 @@ public class GridH2Table extends TableBase {
setColumns(newCols);
desc.refreshMetadataFromTypeDescriptor();
+
+ setModified();
}
finally {
unlock(true);
[26/35] ignite git commit: ignite-3478
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b72f362d/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java b/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java
deleted file mode 100644
index 9351fcc..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java
+++ /dev/null
@@ -1,1750 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.GridAtomicLong;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.NotNull;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- *
- */
-public class MvccTestApp2 {
- /** */
- private static final boolean DEBUG_LOG = false;
-
- /** */
- private static final boolean SQL = false;
-
- public static void main1(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 2, true);
-
- Map<Object, Object> vals = cluster.sqlAll();
-
- System.out.println();
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)getData.get(i);
-
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main0(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- //cluster.txRemoveTransfer(0, 1);
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (Map.Entry<Object, Object> e : getData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main(String[] args) throws Exception {
- final AtomicBoolean err = new AtomicBoolean();
-
- final int READ_THREADS = 4;
- final int UPDATE_THREADS = 4;
- final int ACCOUNTS = 50;
-
- final int START_VAL = 100000;
-
- for (int iter = 0; iter < 1000; iter++) {
- System.out.println("Iteration [readThreads=" + READ_THREADS +
- ", updateThreads=" + UPDATE_THREADS + ", accounts=" + ACCOUNTS + ", iter=" + iter + ']');
-
- final TestCluster cluster = new TestCluster(1);
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- List<Thread> threads = new ArrayList<>();
-
- Thread cleanupThread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("cleanup");
-
- try {
- while (!stop.get()) {
- cluster.cleanup();
-
- Thread.sleep(1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- threads.add(cleanupThread);
-
- cleanupThread.start();
-
- final boolean REMOVES = false;
-
- for (int i = 0; i < READ_THREADS; i++) {
- final int id = i;
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("read" + id);
-
- int cnt = 0;
-
- while (!stop.get()) {
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- cnt++;
-
- int sum = 0;
-
- if (REMOVES) {
- for (Map.Entry<Object, Object> e : qryData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
- else
- System.out.println("With null");
- }
- }
- else {
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- if (val == null) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("No value for key: " + i);
- }
-
- return;
- }
-
- sum += val;
- }
- }
-
- if (sum != ACCOUNTS * START_VAL) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("Invalid get sum: " + sum);
- }
- }
- }
-
- System.out.println("Get cnt: " + cnt);
- }
- catch (Throwable e) {
- e.printStackTrace();
-
- err.set(true);
- stop.set(true);
- }
- }
- });
-
- threads.add(thread);
-
- thread.start();
- }
-
- for (int i = 0; i < UPDATE_THREADS; i++) {
- final int id = i;
-
- Thread thread;
-
- if (REMOVES) {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (rnd.nextBoolean()) {
- //cluster.txRemoveTransfer(id1, id2);
- }
- else
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- }
- else {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (id1 > id2) {
- int tmp = id1;
- id1 = id2;
- id2 = tmp;
- }
-
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- }
-
- threads.add(thread);
-
- thread.start();
- }
-
- long endTime = System.currentTimeMillis() + 60_000;
-
- while (!stop.get()) {
- Thread.sleep(1000);
-
- if (System.currentTimeMillis() >= endTime)
- break;
-
- //cluster.dumpMvccInfo();
- }
-
- stop.set(true);
-
- for (Thread thread : threads)
- thread.join();
-
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- System.out.println("Val " + val);
-
- if (val != null)
- sum += val;
- }
-
- System.out.println("Sum=" + sum + ", expSum=" + (ACCOUNTS * START_VAL));
-
- if (err.get()) {
- System.out.println("Error!");
-
- System.exit(1);
- }
-
-// cluster.dumpMvccInfo();
-//
-// System.out.println("Cleanup");
-//
-// cluster.cleanup();
-//
-// cluster.dumpMvccInfo();
-
- TestDebugLog.clear();
- }
- }
-
- /**
- *
- */
- static class TestCluster {
- /** */
- final List<Node> nodes = new ArrayList<>();
-
- /** */
- final Coordinator crd;
-
- /** */
- final AtomicLong txIdGen = new AtomicLong(10_000);
-
- TestCluster(int nodesNum) {
- crd = new Coordinator();
-
- for (int i = 0; i < nodesNum; i++)
- nodes.add(new Node(i));
- }
-
- void cleanup() {
- CoordinatorCounter cntr = crd.cleanupVersion();
-
- for (Node node : nodes)
- node.dataStore.cleanup(cntr);
- }
-
- void txPutAll(Map<Object, Object> data) {
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- for (Object key : data.keySet()) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- mappedEntries.put(key, node);
- }
-
- TxVersion ver = crd.nextTxCounter(txId);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(ver.cntr, txId);
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), data.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, ver.cntr.cntr);
- }
-
- void txTransfer(Integer id1, Integer id2, boolean fromFirst) throws Exception {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(id1);
- keys.add(id2);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- TxVersion ver = crd.nextTxCounter(txId);
-
- Collection<TxId> waitTxs = null;
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Collection<TxId> txs = node.dataStore.waitTxsAck(key, ver.activeTxs);
-
- if (txs != null) {
- if (waitTxs == null)
- waitTxs = txs;
- else
- waitTxs.addAll(txs);
- }
- }
-
- if (waitTxs != null) {
- crd.waitTxs(waitTxs);
- }
-
-
- Integer curVal1 = (Integer)vals.get(id1);
- Integer curVal2 = (Integer)vals.get(id2);
-
- boolean update = false;
-
- Integer newVal1 = null;
- Integer newVal2 = null;
-
- if (curVal1 != null && curVal2 != null) {
- if (fromFirst) {
- if (curVal1 > 0) {
- update = true;
-
- newVal1 = curVal1 - 1;
- newVal2 = curVal2 + 1;
- }
- }
- else {
- if (curVal2 > 0) {
- update = true;
-
- newVal1 = curVal1 + 1;
- newVal2 = curVal2 - 1;
- }
- }
- }
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(id1, newVal1);
- newVals.put(id2, newVal2);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(ver.cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("update", txId, id1, newVal1, id2, newVal2, ver.cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, ver.cntr.cntr);
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, ver.cntr.cntr);
- }
-
-// if (DEBUG_LOG)
-// TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
-// void txRemoveTransfer(Integer from, Integer to) {
-// TreeSet<Integer> keys = new TreeSet<>();
-//
-// keys.add(from);
-// keys.add(to);
-//
-// TxId txId = new TxId(txIdGen.incrementAndGet());
-//
-// Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-//
-// Map<Object, Object> vals = new HashMap<>();
-//
-// for (Object key : keys) {
-// int nodeIdx = nodeForKey(key);
-//
-// Node node = nodes.get(nodeIdx);
-//
-// node.dataStore.lockEntry(key);
-//
-// vals.put(key, node.dataStore.lastValue(key));
-//
-// mappedEntries.put(key, node);
-// }
-//
-// CoordinatorCounter cntr = crd.nextTxCounter(txId);
-//
-// Integer fromVal = (Integer)vals.get(from);
-// Integer toVal = (Integer)vals.get(to);
-//
-// boolean update = fromVal != null && toVal != null;
-//
-// if (update) {
-// Map<Object, Object> newVals = new HashMap<>();
-//
-// newVals.put(from, null);
-// newVals.put(to, fromVal + toVal);
-//
-// MvccCounter mvccVer = new MvccCounter(cntr, txId);
-//
-// if (DEBUG_LOG) {
-// TestDebugLog.msgs.add(new TestDebugLog.Msg6("remove", txId, from, fromVal, to, toVal, cntr));
-// }
-//
-// for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
-// Node node = e.getValue();
-//
-// node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
-// }
-//
-// for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
-// Node node = e.getValue();
-//
-// node.dataStore.unlockEntry(e.getKey());
-// }
-// }
-// else {
-// for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
-// Node node = e.getValue();
-//
-// node.dataStore.unlockEntry(e.getKey());
-// }
-// }
-//
-// crd.txDone(txId, cntr.cntr);
-//
-// if (DEBUG_LOG)
-// TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
-// }
-
- public void dumpMvccInfo() {
- for (Node node : nodes) {
- int sql = node.dataStore.mvccSqlIdx.size();
-
- for (Map.Entry<Object, MvccValue> e : node.dataStore.mainIdx.entrySet()) {
- List<MvccValue> list = node.dataStore.mvccIdx.get(e.getKey());
-
- int size = 0;
-
- if (list != null) {
- synchronized (list) {
- size = list.size();
- }
- }
-
- System.out.println("Mvcc info [key=" + e.getKey() +
- ", val=" + e.getValue() +
- ", mvccVals=" + size +
- ", sqlVals=" + sql + ']');
- }
- }
- }
-
- public Map<Object, Object> sqlAll() {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Node node : nodes) {
- Map<Object, Object> nodeRes = node.dataStore.sqlQuery(qryVer);
-
- res.putAll(nodeRes);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sqlAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- public Map<Object, Object> getAll(Set<?> keys) {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Object val = node.dataStore.get(key, qryVer);
-
- res.put(key, val);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("getAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- private int nodeForKey(Object key) {
- return U.safeAbs(key.hashCode()) % nodes.size();
- }
- }
-
- /**
- *
- */
- static class Node {
- /** */
- final DataStore dataStore;
-
- /** */
- final int nodexIdx;
-
- public Node(int nodexIdx) {
- this.nodexIdx = nodexIdx;
-
- dataStore = new DataStore();
- }
-
- @Override public String toString() {
- return "Node [idx=" + nodexIdx + ']';
- }
- }
-
- static class TxVersion {
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- public TxVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
- }
-
- /**
- *
- */
- static class Coordinator {
- /** */
- private final AtomicLong cntr = new AtomicLong(-1);
-
- /** */
- private final GridAtomicLong commitCntr = new GridAtomicLong(-1);
-
- /** */
- private final Map<Long, Integer> activeQueries = new ConcurrentHashMap8<>();
-
- /** */
- @GridToStringInclude
- private final ConcurrentHashMap8<TxId, Long> activeTxs = new ConcurrentHashMap8<>();
-
- synchronized void waitTxs(Collection<TxId> waitTxs) throws InterruptedException {
- for (TxId txId : waitTxs) {
- while (activeTxs.containsKey(txId))
- wait();
- }
- }
-
- synchronized TxVersion nextTxCounter(TxId txId) {
- long cur = cntr.get();
-
- activeTxs.put(txId, cur + 1);
-
- CoordinatorCounter newCtr = new CoordinatorCounter(cntr.incrementAndGet());
-
- Set<TxId> txs = new HashSet<>();
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet())
- txs.add(e.getKey());
-
- TxVersion ver = new TxVersion(newCtr, txs);
-
- return ver;
- }
-
- synchronized void txDone(TxId txId, long cntr) {
- Long rmvd = activeTxs.remove(txId);
-
- assert rmvd != null;
-
- commitCntr.setIfGreater(cntr);
-
- notifyAll();
- }
-
- private Long minActive(Set<TxId> txs) {
- Long minActive = null;
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet()) {
- if (txs != null)
- txs.add(e.getKey());
-
-// TxId val = e.getValue();
-//
-// while (val.cntr == -1)
-// Thread.yield();
-
- long cntr = e.getValue();
-
- if (minActive == null)
- minActive = cntr;
- else if (cntr < minActive)
- minActive = cntr;
- }
-
- return minActive;
- }
-
- synchronized MvccQueryVersion queryVersion() {
- long useCntr = commitCntr.get();
-
-// Long minActive = minActive(txs);
-//
-// if (minActive != null && minActive < useCntr)
-// useCntr = minActive - 1;
-
- Set<TxId> txs = new HashSet<>();
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet())
- txs.add(e.getKey());
-
- MvccQueryVersion qryVer = new MvccQueryVersion(new CoordinatorCounter(useCntr), txs);
-
- Integer qryCnt = activeQueries.get(useCntr);
-
- if (qryCnt != null)
- activeQueries.put(useCntr, qryCnt + 1);
- else
- activeQueries.put(useCntr, 1);
-
-
- return qryVer;
- }
-
- synchronized void queryDone(CoordinatorCounter cntr) {
- Integer qryCnt = activeQueries.get(cntr.cntr);
-
- assert qryCnt != null : cntr.cntr;
-
- int left = qryCnt - 1;
-
- assert left >= 0 : left;
-
- if (left == 0)
- activeQueries.remove(cntr.cntr);
- }
-
- synchronized CoordinatorCounter cleanupVersion() {
- long useCntr = commitCntr.get();
-
- Long minActive = minActive(null);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- for (Long qryCntr : activeQueries.keySet()) {
- if (qryCntr <= useCntr)
- useCntr = qryCntr - 1;
- }
-
- return new CoordinatorCounter(useCntr);
- }
-
- @Override public String toString() {
- return S.toString(Coordinator.class, this);
- }
- }
-
- /**
- *
- */
- static class CoordinatorCounter implements Comparable<CoordinatorCounter> {
- /** */
- private final long topVer; // TODO
-
- /** */
- private final long cntr;
-
- CoordinatorCounter(long cntr) {
- this.topVer = 1;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(CoordinatorCounter o) {
- return Long.compare(cntr, o.cntr);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- CoordinatorCounter that = (CoordinatorCounter)o;
-
- return cntr == that.cntr;
- }
-
- @Override public int hashCode() {
- return (int)(cntr ^ (cntr >>> 32));
- }
-
- @Override public String toString() {
- return "Cntr [c=" + cntr + ']';
- }
- }
-
- /**
- *
- */
- static class MvccUpdateVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final TxId txId;
-
- /**
- * @param cntr
- */
- MvccUpdateVersion(CoordinatorCounter cntr, TxId txId) {
- assert cntr != null;
-
- this.cntr = cntr;
- this.txId = txId;
- }
-
- @Override public String toString() {
- return S.toString(MvccUpdateVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccQueryVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- MvccQueryVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
-
- @Override public String toString() {
- return S.toString(MvccQueryVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class TxId {
- /** */
- @GridToStringInclude
- final long id;
-
- TxId(long id) {
- this.id = id;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TxId txId = (TxId) o;
-
- return id == txId.id;
- }
-
- @Override public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override public String toString() {
- return S.toString(TxId.class, this);
- }
- }
-
- /**
- *
- */
- static class SqlKey implements Comparable<SqlKey> {
- /** */
- final Comparable key;
-
- /** */
- final Comparable val;
-
- /** */
- final CoordinatorCounter cntr;
-
- public SqlKey(Object key, Object val, CoordinatorCounter cntr) {
- this.key = (Comparable)key;
- this.val = (Comparable)val;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(@NotNull SqlKey o) {
- int cmp;
-
- if (val != null && o.val != null)
- cmp = val.compareTo(o.val);
- else {
- if (val != null)
- cmp = 1;
- else
- cmp = o.val == null ? 0 : -1;
- }
-
-
- if (cmp == 0) {
- cmp = key.compareTo(o.key);
-
- if (cmp == 0)
- cmp = cntr.compareTo(o.cntr);
- }
-
- return cmp;
- }
-
- @Override public String toString() {
- return "SqlKey [key=" + key + ", val=" + val + ']';
- }
- }
-
- /**
- *
- */
- static class DataStore {
- /** */
- private final ConcurrentHashMap<Object, ReentrantLock> locks = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, MvccValue> mainIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, List<MvccValue>> mvccIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentSkipListMap<SqlKey, MvccSqlValue> mvccSqlIdx = new ConcurrentSkipListMap<>();
-
- void cleanup(CoordinatorCounter cleanupCntr) {
- for (Map.Entry<Object, List<MvccValue>> e : mvccIdx.entrySet()) {
- lockEntry(e.getKey());
-
- try {
- List<MvccValue> list = e.getValue();
-
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (val.ver.cntr.compareTo(cleanupCntr) <= 0) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6_1("cleanup",
- e.getKey(), val.val, val.ver, cleanupCntr.cntr, null, null));
- }
-
- MvccValue prev;
-
- if (val.val != null)
- prev = mainIdx.put(e.getKey(), val);
- else
- prev = mainIdx.remove(e.getKey());
-
- if (prev != null) {
- SqlKey key = new SqlKey(e.getKey(), prev.val, prev.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
-
- for (int j = 0; j <= i; j++) {
- MvccValue rmvd = list.remove(0);
-
- assert rmvd != null;
-
- if (j != i || rmvd.val == null) {
- SqlKey key = new SqlKey(e.getKey(), rmvd.val, rmvd.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
- }
-
- if (list.isEmpty())
- mvccIdx.remove(e.getKey());
-
- break;
- }
- }
- }
- }
- finally {
- unlockEntry(e.getKey());
- }
- }
- }
-
- Collection<TxId> waitTxsAck(Object key, Collection<TxId> activeTxs) {
- if (F.isEmpty(activeTxs))
- return null;
-
- List<MvccValue> list = mvccIdx.get(key);
-
- List<TxId> waitTxs = null;
-
- if (list != null) {
- for (MvccValue val : list) {
- if (activeTxs.contains(val.ver.txId)) {
- if (waitTxs == null)
- waitTxs = new ArrayList<>();
-
- waitTxs.add(val.ver.txId);
- }
- }
- }
-
- return waitTxs;
- }
-
- void lockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.lock();
- }
-
- void unlockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.unlock();
- }
-
- void updateEntry(Object key, Object val, MvccUpdateVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list == null) {
- Object old = mvccIdx.putIfAbsent(key, list = new ArrayList<>());
-
- assert old == null;
- }
-
- MvccValue prevVal = null;
-
- synchronized (list) {
- if (!list.isEmpty())
- prevVal = list.get(list.size() - 1);
-
- list.add(new MvccValue(val, ver));
- }
-
- if (prevVal == null)
- prevVal = mainIdx.get(key);
-
- if (prevVal != null) {
- SqlKey prevKey = new SqlKey(key, prevVal.val, prevVal.ver.cntr);
-
- MvccSqlValue old =
- mvccSqlIdx.put(prevKey, new MvccSqlValue(prevVal.val, prevVal.ver, ver));
-
- assert old != null;
- }
-
- mvccSqlIdx.put(new SqlKey(key, val, ver.cntr), new MvccSqlValue(val, ver, null));
- }
-
- Object lastValue(Object key) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- if (list.size() > 0)
- return list.get(list.size() - 1).val;
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- return val != null ? val.val : null;
- }
-
- Map<Object, Object> sqlQuery(MvccQueryVersion qryVer) {
- Map<Object, Object> res = new HashMap<>();
-
- for (Map.Entry<SqlKey, MvccSqlValue> e : mvccSqlIdx.entrySet()) {
- MvccSqlValue val = e.getValue();
-
- if (!versionVisible(val.ver, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sql skip mvcc val", e.getKey().key, val.val, val.ver));
- }
-
- continue;
- }
-
- MvccUpdateVersion newVer = val.newVer;
-
- if (newVer != null && versionVisible(newVer, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql skip mvcc val2", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- continue;
- }
-
- Object old = res.put(e.getKey().key, e.getValue().val);
-
- if (DEBUG_LOG) {
- //TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql get mvcc val", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- if (old != null) {
- TestDebugLog.printAllAndExit("Already has value for key [key=" + e.getKey().key +
- ", qryVer=" + qryVer +
- ", oldVal=" + old +
- ", newVal=" + e.getValue().val +
- ']');
- }
-
- assert old == null;
- }
-
- return res;
- }
-
- private boolean versionVisible(MvccUpdateVersion ver, MvccQueryVersion qryVer) {
- int cmp = ver.cntr.compareTo(qryVer.cntr);
-
- return cmp <= 0 && !qryVer.activeTxs.contains(ver.txId);
- }
-
- Object get(Object key, MvccQueryVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (!versionVisible(val.ver, ver))
- continue;
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read mvcc val", key, val.val, val.ver));
- }
-
- return val.val;
- }
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- if (val != null) {
- int cmp = val.ver.cntr.compareTo(ver.cntr);
-
- if (DEBUG_LOG) {
- if (cmp > 0) {
- synchronized (TestDebugLog.msgs) {
- TestDebugLog.msgs.add(new TestDebugLog.Message("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver.cntr + ']'));
-
- TestDebugLog.printAllAndExit("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver + ']');
- }
- }
- }
-
- assert cmp <= 0 : "Committed [ver=" + val.ver + ", qryVer=" + ver.cntr + ']';
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted val", key, val, val.ver));
- }
- else {
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted null", key, null, null));
- }
-
- return val != null ? val.val : null;
- }
-
- private ReentrantLock lock(Object key) {
- ReentrantLock e = locks.get(key);
-
- if (e == null) {
- ReentrantLock old = locks.putIfAbsent(key, e = new ReentrantLock());
-
- if (old != null)
- e = old;
- }
-
- return e;
- }
- }
-
- /**
- *
- */
- static class MvccValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- MvccValue(Object val, MvccUpdateVersion ver) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- }
-
- @Override public String toString() {
- return S.toString(MvccValue.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccSqlValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion newVer;
-
- MvccSqlValue(Object val, MvccUpdateVersion ver, MvccUpdateVersion newVer) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- this.newVer = newVer;
- }
-
- @Override public String toString() {
- return S.toString(MvccSqlValue.class, this);
- }
- }
-
- static void log(String msg) {
- System.out.println(Thread.currentThread() + ": " + msg);
- }
-
- static class TestDebugLog {
- /** */
- //static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(1_000_000));
- static final ConcurrentLinkedQueue<Object> msgs = new ConcurrentLinkedQueue<>();
-
-
-
- /** */
- private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
-
- static class Message {
- String thread = Thread.currentThread().getName();
-
- String msg;
-
- long ts = U.currentTimeMillis();
-
- public Message(String msg) {
- this.msg = msg;
- }
-
- public String toString() {
- return "Msg [msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg2 extends Message{
- Object v1;
- Object v2;
-
- public Msg2(String msg, Object v1, Object v2) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", msg=" + msg +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg3 extends Message{
- Object v1;
- Object v2;
- Object v3;
-
- public Msg3(String msg, Object v1, Object v2, Object v3) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg4 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
-
- public Msg4(String msg, Object v1, Object v2, Object v3, Object v4) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", v4=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg6 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", txId=" + v1 +
- ", id1=" + v2 +
- ", v1=" + v3 +
- ", id2=" + v4 +
- ", v2=" + v5 +
- ", cntr=" + v6 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
- static class Msg6_1 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6_1(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", key=" + v1 +
- ", val=" + v2 +
- ", ver=" + v3 +
- ", cleanupC=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class EntryMessage extends Message {
- Object key;
- Object val;
-
- public EntryMessage(Object key, Object val, String msg) {
- super(msg);
-
- this.key = key;
- this.val = val;
- }
-
- public String toString() {
- return "EntryMsg [key=" + key + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class PartMessage extends Message {
- int p;
- Object val;
-
- public PartMessage(int p, Object val, String msg) {
- super(msg);
-
- this.p = p;
- this.val = val;
- }
-
- public String toString() {
- return "PartMessage [p=" + p + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static final boolean out = false;
-
- public static void addMessage(String msg) {
- msgs.add(new Message(msg));
-
- if (out)
- System.out.println(msg);
- }
-
- public static void addEntryMessage(Object key, Object val, String msg) {
- if (key instanceof KeyCacheObject)
- key = ((KeyCacheObject)key).value(null, false);
-
- EntryMessage msg0 = new EntryMessage(key, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- public static void addPartMessage(int p, Object val, String msg) {
- PartMessage msg0 = new PartMessage(p, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- static void printAllAndExit(String msg) {
- System.out.println(msg);
-
- TestDebugLog.addMessage(msg);
-
- List<Object> msgs = TestDebugLog.printMessages(true, null);
-
- TestDebugLog.printMessages0(msgs, "test_debug_update.txt");
-
- TestDebugLog.printMessagesForThread(msgs, Thread.currentThread().getName(), "test_debug_thread.txt");
-
- System.exit(1);
- }
-
- public static void printMessagesForThread(List<Object> msgs0, String thread0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String thread = ((Message) msg).thread;
-
- if (thread.equals(thread0))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static void printMessages0(List<Object> msgs0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String msg0 = ((Message) msg).msg;
-
- if (msg0.equals("tx done") || msg0.equals("update") || msg0.equals("cleanup"))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static List<Object> printMessages(boolean file, Integer part) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (part != null && msg instanceof PartMessage) {
- if (((PartMessage) msg).p != part)
- continue;
- }
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0)
- System.out.println(msg);
- }
-
- return msgs0;
- }
-
- public static void printKeyMessages(boolean file, Object key) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- System.out.println(msg);
- }
- }
- }
-
- public static void clear() {
- msgs.clear();
- }
-
- public static void clearEntries() {
- for (Iterator it = msgs.iterator(); it.hasNext();) {
- Object msg = it.next();
-
- if (msg instanceof EntryMessage)
- it.remove();
- }
- }
-
- }}
[23/35] ignite git commit: ignite-3479
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java b/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java
deleted file mode 100644
index 9351fcc..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp2.java
+++ /dev/null
@@ -1,1750 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.GridAtomicLong;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.NotNull;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- *
- */
-public class MvccTestApp2 {
- /** */
- private static final boolean DEBUG_LOG = false;
-
- /** */
- private static final boolean SQL = false;
-
- public static void main1(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 2, true);
-
- Map<Object, Object> vals = cluster.sqlAll();
-
- System.out.println();
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)getData.get(i);
-
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main0(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- //cluster.txRemoveTransfer(0, 1);
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (Map.Entry<Object, Object> e : getData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main(String[] args) throws Exception {
- final AtomicBoolean err = new AtomicBoolean();
-
- final int READ_THREADS = 4;
- final int UPDATE_THREADS = 4;
- final int ACCOUNTS = 50;
-
- final int START_VAL = 100000;
-
- for (int iter = 0; iter < 1000; iter++) {
- System.out.println("Iteration [readThreads=" + READ_THREADS +
- ", updateThreads=" + UPDATE_THREADS + ", accounts=" + ACCOUNTS + ", iter=" + iter + ']');
-
- final TestCluster cluster = new TestCluster(1);
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- List<Thread> threads = new ArrayList<>();
-
- Thread cleanupThread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("cleanup");
-
- try {
- while (!stop.get()) {
- cluster.cleanup();
-
- Thread.sleep(1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- threads.add(cleanupThread);
-
- cleanupThread.start();
-
- final boolean REMOVES = false;
-
- for (int i = 0; i < READ_THREADS; i++) {
- final int id = i;
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("read" + id);
-
- int cnt = 0;
-
- while (!stop.get()) {
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- cnt++;
-
- int sum = 0;
-
- if (REMOVES) {
- for (Map.Entry<Object, Object> e : qryData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
- else
- System.out.println("With null");
- }
- }
- else {
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- if (val == null) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("No value for key: " + i);
- }
-
- return;
- }
-
- sum += val;
- }
- }
-
- if (sum != ACCOUNTS * START_VAL) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("Invalid get sum: " + sum);
- }
- }
- }
-
- System.out.println("Get cnt: " + cnt);
- }
- catch (Throwable e) {
- e.printStackTrace();
-
- err.set(true);
- stop.set(true);
- }
- }
- });
-
- threads.add(thread);
-
- thread.start();
- }
-
- for (int i = 0; i < UPDATE_THREADS; i++) {
- final int id = i;
-
- Thread thread;
-
- if (REMOVES) {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (rnd.nextBoolean()) {
- //cluster.txRemoveTransfer(id1, id2);
- }
- else
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- }
- else {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (id1 > id2) {
- int tmp = id1;
- id1 = id2;
- id2 = tmp;
- }
-
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
- }
-
- threads.add(thread);
-
- thread.start();
- }
-
- long endTime = System.currentTimeMillis() + 60_000;
-
- while (!stop.get()) {
- Thread.sleep(1000);
-
- if (System.currentTimeMillis() >= endTime)
- break;
-
- //cluster.dumpMvccInfo();
- }
-
- stop.set(true);
-
- for (Thread thread : threads)
- thread.join();
-
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- System.out.println("Val " + val);
-
- if (val != null)
- sum += val;
- }
-
- System.out.println("Sum=" + sum + ", expSum=" + (ACCOUNTS * START_VAL));
-
- if (err.get()) {
- System.out.println("Error!");
-
- System.exit(1);
- }
-
-// cluster.dumpMvccInfo();
-//
-// System.out.println("Cleanup");
-//
-// cluster.cleanup();
-//
-// cluster.dumpMvccInfo();
-
- TestDebugLog.clear();
- }
- }
-
- /**
- *
- */
- static class TestCluster {
- /** */
- final List<Node> nodes = new ArrayList<>();
-
- /** */
- final Coordinator crd;
-
- /** */
- final AtomicLong txIdGen = new AtomicLong(10_000);
-
- TestCluster(int nodesNum) {
- crd = new Coordinator();
-
- for (int i = 0; i < nodesNum; i++)
- nodes.add(new Node(i));
- }
-
- void cleanup() {
- CoordinatorCounter cntr = crd.cleanupVersion();
-
- for (Node node : nodes)
- node.dataStore.cleanup(cntr);
- }
-
- void txPutAll(Map<Object, Object> data) {
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- for (Object key : data.keySet()) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- mappedEntries.put(key, node);
- }
-
- TxVersion ver = crd.nextTxCounter(txId);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(ver.cntr, txId);
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), data.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, ver.cntr.cntr);
- }
-
- void txTransfer(Integer id1, Integer id2, boolean fromFirst) throws Exception {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(id1);
- keys.add(id2);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- TxVersion ver = crd.nextTxCounter(txId);
-
- Collection<TxId> waitTxs = null;
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Collection<TxId> txs = node.dataStore.waitTxsAck(key, ver.activeTxs);
-
- if (txs != null) {
- if (waitTxs == null)
- waitTxs = txs;
- else
- waitTxs.addAll(txs);
- }
- }
-
- if (waitTxs != null) {
- crd.waitTxs(waitTxs);
- }
-
-
- Integer curVal1 = (Integer)vals.get(id1);
- Integer curVal2 = (Integer)vals.get(id2);
-
- boolean update = false;
-
- Integer newVal1 = null;
- Integer newVal2 = null;
-
- if (curVal1 != null && curVal2 != null) {
- if (fromFirst) {
- if (curVal1 > 0) {
- update = true;
-
- newVal1 = curVal1 - 1;
- newVal2 = curVal2 + 1;
- }
- }
- else {
- if (curVal2 > 0) {
- update = true;
-
- newVal1 = curVal1 + 1;
- newVal2 = curVal2 - 1;
- }
- }
- }
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(id1, newVal1);
- newVals.put(id2, newVal2);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(ver.cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("update", txId, id1, newVal1, id2, newVal2, ver.cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, ver.cntr.cntr);
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, ver.cntr.cntr);
- }
-
-// if (DEBUG_LOG)
-// TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
-// void txRemoveTransfer(Integer from, Integer to) {
-// TreeSet<Integer> keys = new TreeSet<>();
-//
-// keys.add(from);
-// keys.add(to);
-//
-// TxId txId = new TxId(txIdGen.incrementAndGet());
-//
-// Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-//
-// Map<Object, Object> vals = new HashMap<>();
-//
-// for (Object key : keys) {
-// int nodeIdx = nodeForKey(key);
-//
-// Node node = nodes.get(nodeIdx);
-//
-// node.dataStore.lockEntry(key);
-//
-// vals.put(key, node.dataStore.lastValue(key));
-//
-// mappedEntries.put(key, node);
-// }
-//
-// CoordinatorCounter cntr = crd.nextTxCounter(txId);
-//
-// Integer fromVal = (Integer)vals.get(from);
-// Integer toVal = (Integer)vals.get(to);
-//
-// boolean update = fromVal != null && toVal != null;
-//
-// if (update) {
-// Map<Object, Object> newVals = new HashMap<>();
-//
-// newVals.put(from, null);
-// newVals.put(to, fromVal + toVal);
-//
-// MvccCounter mvccVer = new MvccCounter(cntr, txId);
-//
-// if (DEBUG_LOG) {
-// TestDebugLog.msgs.add(new TestDebugLog.Msg6("remove", txId, from, fromVal, to, toVal, cntr));
-// }
-//
-// for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
-// Node node = e.getValue();
-//
-// node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
-// }
-//
-// for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
-// Node node = e.getValue();
-//
-// node.dataStore.unlockEntry(e.getKey());
-// }
-// }
-// else {
-// for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
-// Node node = e.getValue();
-//
-// node.dataStore.unlockEntry(e.getKey());
-// }
-// }
-//
-// crd.txDone(txId, cntr.cntr);
-//
-// if (DEBUG_LOG)
-// TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
-// }
-
- public void dumpMvccInfo() {
- for (Node node : nodes) {
- int sql = node.dataStore.mvccSqlIdx.size();
-
- for (Map.Entry<Object, MvccValue> e : node.dataStore.mainIdx.entrySet()) {
- List<MvccValue> list = node.dataStore.mvccIdx.get(e.getKey());
-
- int size = 0;
-
- if (list != null) {
- synchronized (list) {
- size = list.size();
- }
- }
-
- System.out.println("Mvcc info [key=" + e.getKey() +
- ", val=" + e.getValue() +
- ", mvccVals=" + size +
- ", sqlVals=" + sql + ']');
- }
- }
- }
-
- public Map<Object, Object> sqlAll() {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Node node : nodes) {
- Map<Object, Object> nodeRes = node.dataStore.sqlQuery(qryVer);
-
- res.putAll(nodeRes);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sqlAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- public Map<Object, Object> getAll(Set<?> keys) {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Object val = node.dataStore.get(key, qryVer);
-
- res.put(key, val);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("getAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- private int nodeForKey(Object key) {
- return U.safeAbs(key.hashCode()) % nodes.size();
- }
- }
-
- /**
- *
- */
- static class Node {
- /** */
- final DataStore dataStore;
-
- /** */
- final int nodexIdx;
-
- public Node(int nodexIdx) {
- this.nodexIdx = nodexIdx;
-
- dataStore = new DataStore();
- }
-
- @Override public String toString() {
- return "Node [idx=" + nodexIdx + ']';
- }
- }
-
- static class TxVersion {
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- public TxVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
- }
-
- /**
- *
- */
- static class Coordinator {
- /** */
- private final AtomicLong cntr = new AtomicLong(-1);
-
- /** */
- private final GridAtomicLong commitCntr = new GridAtomicLong(-1);
-
- /** */
- private final Map<Long, Integer> activeQueries = new ConcurrentHashMap8<>();
-
- /** */
- @GridToStringInclude
- private final ConcurrentHashMap8<TxId, Long> activeTxs = new ConcurrentHashMap8<>();
-
- synchronized void waitTxs(Collection<TxId> waitTxs) throws InterruptedException {
- for (TxId txId : waitTxs) {
- while (activeTxs.containsKey(txId))
- wait();
- }
- }
-
- synchronized TxVersion nextTxCounter(TxId txId) {
- long cur = cntr.get();
-
- activeTxs.put(txId, cur + 1);
-
- CoordinatorCounter newCtr = new CoordinatorCounter(cntr.incrementAndGet());
-
- Set<TxId> txs = new HashSet<>();
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet())
- txs.add(e.getKey());
-
- TxVersion ver = new TxVersion(newCtr, txs);
-
- return ver;
- }
-
- synchronized void txDone(TxId txId, long cntr) {
- Long rmvd = activeTxs.remove(txId);
-
- assert rmvd != null;
-
- commitCntr.setIfGreater(cntr);
-
- notifyAll();
- }
-
- private Long minActive(Set<TxId> txs) {
- Long minActive = null;
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet()) {
- if (txs != null)
- txs.add(e.getKey());
-
-// TxId val = e.getValue();
-//
-// while (val.cntr == -1)
-// Thread.yield();
-
- long cntr = e.getValue();
-
- if (minActive == null)
- minActive = cntr;
- else if (cntr < minActive)
- minActive = cntr;
- }
-
- return minActive;
- }
-
- synchronized MvccQueryVersion queryVersion() {
- long useCntr = commitCntr.get();
-
-// Long minActive = minActive(txs);
-//
-// if (minActive != null && minActive < useCntr)
-// useCntr = minActive - 1;
-
- Set<TxId> txs = new HashSet<>();
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet())
- txs.add(e.getKey());
-
- MvccQueryVersion qryVer = new MvccQueryVersion(new CoordinatorCounter(useCntr), txs);
-
- Integer qryCnt = activeQueries.get(useCntr);
-
- if (qryCnt != null)
- activeQueries.put(useCntr, qryCnt + 1);
- else
- activeQueries.put(useCntr, 1);
-
-
- return qryVer;
- }
-
- synchronized void queryDone(CoordinatorCounter cntr) {
- Integer qryCnt = activeQueries.get(cntr.cntr);
-
- assert qryCnt != null : cntr.cntr;
-
- int left = qryCnt - 1;
-
- assert left >= 0 : left;
-
- if (left == 0)
- activeQueries.remove(cntr.cntr);
- }
-
- synchronized CoordinatorCounter cleanupVersion() {
- long useCntr = commitCntr.get();
-
- Long minActive = minActive(null);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- for (Long qryCntr : activeQueries.keySet()) {
- if (qryCntr <= useCntr)
- useCntr = qryCntr - 1;
- }
-
- return new CoordinatorCounter(useCntr);
- }
-
- @Override public String toString() {
- return S.toString(Coordinator.class, this);
- }
- }
-
- /**
- *
- */
- static class CoordinatorCounter implements Comparable<CoordinatorCounter> {
- /** */
- private final long topVer; // TODO
-
- /** */
- private final long cntr;
-
- CoordinatorCounter(long cntr) {
- this.topVer = 1;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(CoordinatorCounter o) {
- return Long.compare(cntr, o.cntr);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- CoordinatorCounter that = (CoordinatorCounter)o;
-
- return cntr == that.cntr;
- }
-
- @Override public int hashCode() {
- return (int)(cntr ^ (cntr >>> 32));
- }
-
- @Override public String toString() {
- return "Cntr [c=" + cntr + ']';
- }
- }
-
- /**
- *
- */
- static class MvccUpdateVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final TxId txId;
-
- /**
- * @param cntr
- */
- MvccUpdateVersion(CoordinatorCounter cntr, TxId txId) {
- assert cntr != null;
-
- this.cntr = cntr;
- this.txId = txId;
- }
-
- @Override public String toString() {
- return S.toString(MvccUpdateVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccQueryVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- MvccQueryVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
-
- @Override public String toString() {
- return S.toString(MvccQueryVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class TxId {
- /** */
- @GridToStringInclude
- final long id;
-
- TxId(long id) {
- this.id = id;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TxId txId = (TxId) o;
-
- return id == txId.id;
- }
-
- @Override public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override public String toString() {
- return S.toString(TxId.class, this);
- }
- }
-
- /**
- *
- */
- static class SqlKey implements Comparable<SqlKey> {
- /** */
- final Comparable key;
-
- /** */
- final Comparable val;
-
- /** */
- final CoordinatorCounter cntr;
-
- public SqlKey(Object key, Object val, CoordinatorCounter cntr) {
- this.key = (Comparable)key;
- this.val = (Comparable)val;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(@NotNull SqlKey o) {
- int cmp;
-
- if (val != null && o.val != null)
- cmp = val.compareTo(o.val);
- else {
- if (val != null)
- cmp = 1;
- else
- cmp = o.val == null ? 0 : -1;
- }
-
-
- if (cmp == 0) {
- cmp = key.compareTo(o.key);
-
- if (cmp == 0)
- cmp = cntr.compareTo(o.cntr);
- }
-
- return cmp;
- }
-
- @Override public String toString() {
- return "SqlKey [key=" + key + ", val=" + val + ']';
- }
- }
-
- /**
- *
- */
- static class DataStore {
- /** */
- private final ConcurrentHashMap<Object, ReentrantLock> locks = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, MvccValue> mainIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, List<MvccValue>> mvccIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentSkipListMap<SqlKey, MvccSqlValue> mvccSqlIdx = new ConcurrentSkipListMap<>();
-
- void cleanup(CoordinatorCounter cleanupCntr) {
- for (Map.Entry<Object, List<MvccValue>> e : mvccIdx.entrySet()) {
- lockEntry(e.getKey());
-
- try {
- List<MvccValue> list = e.getValue();
-
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (val.ver.cntr.compareTo(cleanupCntr) <= 0) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6_1("cleanup",
- e.getKey(), val.val, val.ver, cleanupCntr.cntr, null, null));
- }
-
- MvccValue prev;
-
- if (val.val != null)
- prev = mainIdx.put(e.getKey(), val);
- else
- prev = mainIdx.remove(e.getKey());
-
- if (prev != null) {
- SqlKey key = new SqlKey(e.getKey(), prev.val, prev.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
-
- for (int j = 0; j <= i; j++) {
- MvccValue rmvd = list.remove(0);
-
- assert rmvd != null;
-
- if (j != i || rmvd.val == null) {
- SqlKey key = new SqlKey(e.getKey(), rmvd.val, rmvd.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
- }
-
- if (list.isEmpty())
- mvccIdx.remove(e.getKey());
-
- break;
- }
- }
- }
- }
- finally {
- unlockEntry(e.getKey());
- }
- }
- }
-
- Collection<TxId> waitTxsAck(Object key, Collection<TxId> activeTxs) {
- if (F.isEmpty(activeTxs))
- return null;
-
- List<MvccValue> list = mvccIdx.get(key);
-
- List<TxId> waitTxs = null;
-
- if (list != null) {
- for (MvccValue val : list) {
- if (activeTxs.contains(val.ver.txId)) {
- if (waitTxs == null)
- waitTxs = new ArrayList<>();
-
- waitTxs.add(val.ver.txId);
- }
- }
- }
-
- return waitTxs;
- }
-
- void lockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.lock();
- }
-
- void unlockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.unlock();
- }
-
- void updateEntry(Object key, Object val, MvccUpdateVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list == null) {
- Object old = mvccIdx.putIfAbsent(key, list = new ArrayList<>());
-
- assert old == null;
- }
-
- MvccValue prevVal = null;
-
- synchronized (list) {
- if (!list.isEmpty())
- prevVal = list.get(list.size() - 1);
-
- list.add(new MvccValue(val, ver));
- }
-
- if (prevVal == null)
- prevVal = mainIdx.get(key);
-
- if (prevVal != null) {
- SqlKey prevKey = new SqlKey(key, prevVal.val, prevVal.ver.cntr);
-
- MvccSqlValue old =
- mvccSqlIdx.put(prevKey, new MvccSqlValue(prevVal.val, prevVal.ver, ver));
-
- assert old != null;
- }
-
- mvccSqlIdx.put(new SqlKey(key, val, ver.cntr), new MvccSqlValue(val, ver, null));
- }
-
- Object lastValue(Object key) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- if (list.size() > 0)
- return list.get(list.size() - 1).val;
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- return val != null ? val.val : null;
- }
-
- Map<Object, Object> sqlQuery(MvccQueryVersion qryVer) {
- Map<Object, Object> res = new HashMap<>();
-
- for (Map.Entry<SqlKey, MvccSqlValue> e : mvccSqlIdx.entrySet()) {
- MvccSqlValue val = e.getValue();
-
- if (!versionVisible(val.ver, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sql skip mvcc val", e.getKey().key, val.val, val.ver));
- }
-
- continue;
- }
-
- MvccUpdateVersion newVer = val.newVer;
-
- if (newVer != null && versionVisible(newVer, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql skip mvcc val2", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- continue;
- }
-
- Object old = res.put(e.getKey().key, e.getValue().val);
-
- if (DEBUG_LOG) {
- //TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql get mvcc val", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- if (old != null) {
- TestDebugLog.printAllAndExit("Already has value for key [key=" + e.getKey().key +
- ", qryVer=" + qryVer +
- ", oldVal=" + old +
- ", newVal=" + e.getValue().val +
- ']');
- }
-
- assert old == null;
- }
-
- return res;
- }
-
- private boolean versionVisible(MvccUpdateVersion ver, MvccQueryVersion qryVer) {
- int cmp = ver.cntr.compareTo(qryVer.cntr);
-
- return cmp <= 0 && !qryVer.activeTxs.contains(ver.txId);
- }
-
- Object get(Object key, MvccQueryVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (!versionVisible(val.ver, ver))
- continue;
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read mvcc val", key, val.val, val.ver));
- }
-
- return val.val;
- }
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- if (val != null) {
- int cmp = val.ver.cntr.compareTo(ver.cntr);
-
- if (DEBUG_LOG) {
- if (cmp > 0) {
- synchronized (TestDebugLog.msgs) {
- TestDebugLog.msgs.add(new TestDebugLog.Message("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver.cntr + ']'));
-
- TestDebugLog.printAllAndExit("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver + ']');
- }
- }
- }
-
- assert cmp <= 0 : "Committed [ver=" + val.ver + ", qryVer=" + ver.cntr + ']';
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted val", key, val, val.ver));
- }
- else {
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted null", key, null, null));
- }
-
- return val != null ? val.val : null;
- }
-
- private ReentrantLock lock(Object key) {
- ReentrantLock e = locks.get(key);
-
- if (e == null) {
- ReentrantLock old = locks.putIfAbsent(key, e = new ReentrantLock());
-
- if (old != null)
- e = old;
- }
-
- return e;
- }
- }
-
- /**
- *
- */
- static class MvccValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- MvccValue(Object val, MvccUpdateVersion ver) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- }
-
- @Override public String toString() {
- return S.toString(MvccValue.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccSqlValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion newVer;
-
- MvccSqlValue(Object val, MvccUpdateVersion ver, MvccUpdateVersion newVer) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- this.newVer = newVer;
- }
-
- @Override public String toString() {
- return S.toString(MvccSqlValue.class, this);
- }
- }
-
- static void log(String msg) {
- System.out.println(Thread.currentThread() + ": " + msg);
- }
-
- static class TestDebugLog {
- /** */
- //static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(1_000_000));
- static final ConcurrentLinkedQueue<Object> msgs = new ConcurrentLinkedQueue<>();
-
-
-
- /** */
- private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
-
- static class Message {
- String thread = Thread.currentThread().getName();
-
- String msg;
-
- long ts = U.currentTimeMillis();
-
- public Message(String msg) {
- this.msg = msg;
- }
-
- public String toString() {
- return "Msg [msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg2 extends Message{
- Object v1;
- Object v2;
-
- public Msg2(String msg, Object v1, Object v2) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", msg=" + msg +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg3 extends Message{
- Object v1;
- Object v2;
- Object v3;
-
- public Msg3(String msg, Object v1, Object v2, Object v3) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg4 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
-
- public Msg4(String msg, Object v1, Object v2, Object v3, Object v4) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", v4=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg6 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", txId=" + v1 +
- ", id1=" + v2 +
- ", v1=" + v3 +
- ", id2=" + v4 +
- ", v2=" + v5 +
- ", cntr=" + v6 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
- static class Msg6_1 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6_1(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", key=" + v1 +
- ", val=" + v2 +
- ", ver=" + v3 +
- ", cleanupC=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class EntryMessage extends Message {
- Object key;
- Object val;
-
- public EntryMessage(Object key, Object val, String msg) {
- super(msg);
-
- this.key = key;
- this.val = val;
- }
-
- public String toString() {
- return "EntryMsg [key=" + key + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class PartMessage extends Message {
- int p;
- Object val;
-
- public PartMessage(int p, Object val, String msg) {
- super(msg);
-
- this.p = p;
- this.val = val;
- }
-
- public String toString() {
- return "PartMessage [p=" + p + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static final boolean out = false;
-
- public static void addMessage(String msg) {
- msgs.add(new Message(msg));
-
- if (out)
- System.out.println(msg);
- }
-
- public static void addEntryMessage(Object key, Object val, String msg) {
- if (key instanceof KeyCacheObject)
- key = ((KeyCacheObject)key).value(null, false);
-
- EntryMessage msg0 = new EntryMessage(key, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- public static void addPartMessage(int p, Object val, String msg) {
- PartMessage msg0 = new PartMessage(p, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- static void printAllAndExit(String msg) {
- System.out.println(msg);
-
- TestDebugLog.addMessage(msg);
-
- List<Object> msgs = TestDebugLog.printMessages(true, null);
-
- TestDebugLog.printMessages0(msgs, "test_debug_update.txt");
-
- TestDebugLog.printMessagesForThread(msgs, Thread.currentThread().getName(), "test_debug_thread.txt");
-
- System.exit(1);
- }
-
- public static void printMessagesForThread(List<Object> msgs0, String thread0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String thread = ((Message) msg).thread;
-
- if (thread.equals(thread0))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static void printMessages0(List<Object> msgs0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String msg0 = ((Message) msg).msg;
-
- if (msg0.equals("tx done") || msg0.equals("update") || msg0.equals("cleanup"))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static List<Object> printMessages(boolean file, Integer part) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (part != null && msg instanceof PartMessage) {
- if (((PartMessage) msg).p != part)
- continue;
- }
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0)
- System.out.println(msg);
- }
-
- return msgs0;
- }
-
- public static void printKeyMessages(boolean file, Object key) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- System.out.println(msg);
- }
- }
- }
-
- public static void clear() {
- msgs.clear();
- }
-
- public static void clearEntries() {
- for (Iterator it = msgs.iterator(); it.hasNext();) {
- Object msg = it.next();
-
- if (msg instanceof EntryMessage)
- it.remove();
- }
- }
-
- }}
[27/35] ignite git commit: ignite-3478
Posted by sb...@apache.org.
ignite-3478
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b72f362d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b72f362d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b72f362d
Branch: refs/heads/ignite-3479
Commit: b72f362d3d9c25a7f53a19a81a8294fe247f454d
Parents: 666a6ac
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 13:08:36 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 13:08:36 2017 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/MvccTestApp.java | 1689 -----------------
.../apache/ignite/internal/MvccTestApp2.java | 1750 ------------------
.../apache/ignite/internal/MvccTestApp3.java | 1713 -----------------
3 files changed, 5152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/b72f362d/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java b/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java
deleted file mode 100644
index d384339..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java
+++ /dev/null
@@ -1,1689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.GridAtomicLong;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.NotNull;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- *
- */
-public class MvccTestApp {
- /** */
- private static final boolean DEBUG_LOG = false;
-
- /** */
- private static final boolean SQL = false;
-
- public static void main1(String[] args) throws Exception {
- final MvccTestApp.TestCluster cluster = new MvccTestApp.TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 2, true);
-
- Map<Object, Object> vals = cluster.sqlAll();
-
- System.out.println();
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)getData.get(i);
-
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main0(String[] args) throws Exception {
- final MvccTestApp.TestCluster cluster = new MvccTestApp.TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txRemoveTransfer(0, 1);
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (Map.Entry<Object, Object> e : getData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main(String[] args) throws Exception {
- final AtomicBoolean err = new AtomicBoolean();
-
- final int READ_THREADS = 4;
- final int UPDATE_THREADS = 4;
- final int ACCOUNTS = 50;
-
- final int START_VAL = 100000;
-
- for (int iter = 0; iter < 1000; iter++) {
- System.out.println("Iteration [readThreads=" + READ_THREADS +
- ", updateThreads=" + UPDATE_THREADS + ", accounts=" + ACCOUNTS + ", iter=" + iter + ']');
-
- final TestCluster cluster = new TestCluster(1);
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- List<Thread> threads = new ArrayList<>();
-
- Thread cleanupThread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("cleanup");
-
- try {
- while (!stop.get()) {
- cluster.cleanup();
-
- Thread.sleep(1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- threads.add(cleanupThread);
-
- cleanupThread.start();
-
- final boolean REMOVES = false;
-
- for (int i = 0; i < READ_THREADS; i++) {
- final int id = i;
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("read" + id);
-
- int cnt = 0;
-
- while (!stop.get()) {
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- cnt++;
-
- int sum = 0;
-
- if (REMOVES) {
- for (Map.Entry<Object, Object> e : qryData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
- else
- System.out.println("With null");
- }
- }
- else {
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- if (val == null) {
- if (stop.compareAndSet(false, true)) {
- stop.set(true);
- err.set(true);
-
- TestDebugLog.printAllAndExit("No value for key: " + i);
- }
- }
-
- sum += val;
- }
- }
-
- if (sum != ACCOUNTS * START_VAL) {
- if (stop.compareAndSet(false, true)) {
- stop.set(true);
- err.set(true);
-
- TestDebugLog.printAllAndExit("Invalid get sum: " + sum);
- }
- }
-
-// if (cnt % 100 == 0)
-// System.out.println("get " + cnt);
- }
-
- System.out.println("Get cnt: " + cnt);
- }
- });
-
- threads.add(thread);
-
- thread.start();
- }
-
- for (int i = 0; i < UPDATE_THREADS; i++) {
- final int id = i;
-
- Thread thread;
-
- if (REMOVES) {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (rnd.nextBoolean()) {
- cluster.txRemoveTransfer(id1, id2);
- }
- else
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
- else {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (id1 > id2) {
- int tmp = id1;
- id1 = id2;
- id2 = tmp;
- }
-
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
-
- threads.add(thread);
-
- thread.start();
- }
-
- long endTime = System.currentTimeMillis() + 2_000;
-
- while (!stop.get()) {
- Thread.sleep(1000);
-
- if (System.currentTimeMillis() >= endTime)
- break;
-
- //cluster.dumpMvccInfo();
- }
-
- stop.set(true);
-
- for (Thread thread : threads)
- thread.join();
-
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- System.out.println("Val " + val);
-
- if (val != null)
- sum += val;
- }
-
- System.out.println("Sum=" + sum + ", expSum=" + (ACCOUNTS * START_VAL));
-
- if (err.get()) {
- System.out.println("Error!");
-
- System.exit(1);
- }
-
-// cluster.dumpMvccInfo();
-//
-// System.out.println("Cleanup");
-//
-// cluster.cleanup();
-//
-// cluster.dumpMvccInfo();
-
- TestDebugLog.clear();
- }
- }
-
- /**
- *
- */
- static class TestCluster {
- /** */
- final List<Node> nodes = new ArrayList<>();
-
- /** */
- final Coordinator crd;
-
- /** */
- final AtomicLong txIdGen = new AtomicLong(10_000);
-
- TestCluster(int nodesNum) {
- crd = new Coordinator();
-
- for (int i = 0; i < nodesNum; i++)
- nodes.add(new Node(i));
- }
-
- void cleanup() {
- CoordinatorCounter cntr = crd.cleanupVersion();
-
- for (Node node : nodes)
- node.dataStore.cleanup(cntr);
- }
-
- void txPutAll(Map<Object, Object> data) {
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- for (Object key : data.keySet()) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), data.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId);
- }
-
- void txTransfer(Integer id1, Integer id2, boolean fromFirst) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(id1);
- keys.add(id2);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer curVal1 = (Integer)vals.get(id1);
- Integer curVal2 = (Integer)vals.get(id2);
-
- boolean update = false;
-
- Integer newVal1 = null;
- Integer newVal2 = null;
-
- if (curVal1 != null && curVal2 != null) {
- if (fromFirst) {
- if (curVal1 > 0) {
- update = true;
-
- newVal1 = curVal1 - 1;
- newVal2 = curVal2 + 1;
- }
- }
- else {
- if (curVal2 > 0) {
- update = true;
-
- newVal1 = curVal1 + 1;
- newVal2 = curVal2 - 1;
- }
- }
- }
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(id1, newVal1);
- newVals.put(id2, newVal2);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("update", txId, id1, newVal1, id2, newVal2, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId);
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- void txRemoveTransfer(Integer from, Integer to) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(from);
- keys.add(to);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer fromVal = (Integer)vals.get(from);
- Integer toVal = (Integer)vals.get(to);
-
- boolean update = fromVal != null && toVal != null;
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(from, null);
- newVals.put(to, fromVal + toVal);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("remove", txId, from, fromVal, to, toVal, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId);
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- public void dumpMvccInfo() {
- for (Node node : nodes) {
- int sql = node.dataStore.mvccSqlIdx.size();
-
- for (Map.Entry<Object, MvccValue> e : node.dataStore.mainIdx.entrySet()) {
- List<MvccValue> list = node.dataStore.mvccIdx.get(e.getKey());
-
- int size = 0;
-
- if (list != null) {
- synchronized (list) {
- size = list.size();
- }
- }
-
- System.out.println("Mvcc info [key=" + e.getKey() +
- ", val=" + e.getValue() +
- ", mvccVals=" + size +
- ", sqlVals=" + sql + ']');
- }
- }
- }
-
- public Map<Object, Object> sqlAll() {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Node node : nodes) {
- Map<Object, Object> nodeRes = node.dataStore.sqlQuery(qryVer);
-
- res.putAll(nodeRes);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sqlAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- public Map<Object, Object> getAll(Set<?> keys) {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Object val = node.dataStore.get(key, qryVer);
-
- res.put(key, val);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("getAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- private int nodeForKey(Object key) {
- return U.safeAbs(key.hashCode()) % nodes.size();
- }
- }
-
- /**
- *
- */
- static class Node {
- /** */
- final DataStore dataStore;
-
- /** */
- final int nodexIdx;
-
- public Node(int nodexIdx) {
- this.nodexIdx = nodexIdx;
-
- dataStore = new DataStore();
- }
-
- @Override public String toString() {
- return "Node [idx=" + nodexIdx + ']';
- }
- }
-
- /**
- *
- */
- static class Coordinator {
- /** */
- private final AtomicLong cntr = new AtomicLong(-1);
-
- /** */
- private final GridAtomicLong commitCntr = new GridAtomicLong(-1);
-
- /** */
- private final ConcurrentHashMap8<Long, QueryCounter> activeQueries = new ConcurrentHashMap8<>();
-
- /** */
- @GridToStringInclude
- private final ConcurrentHashMap8<TxId, TxId> activeTxs = new ConcurrentHashMap8<>();
-
- CoordinatorCounter nextTxCounter(TxId txId) {
- activeTxs.put(txId, txId);
-
- CoordinatorCounter newCtr = new CoordinatorCounter(cntr.incrementAndGet());
-
- txId.cntr = newCtr.cntr;
-
- return newCtr;
- }
-
- void txDone(TxId txId) {
- TxId cntr = activeTxs.remove(txId);
-
- assert cntr != null && cntr.cntr != -1L;
-
- commitCntr.setIfGreater(cntr.cntr);
- }
-
- private Long minActive(Set<TxId> txs) {
- Long minActive = null;
-
- for (Map.Entry<TxId, TxId> e : activeTxs.entrySet()) {
- if (txs != null)
- txs.add(e.getKey());
-
- TxId val = e.getValue();
-
- while (val.cntr == -1)
- Thread.yield();
-
- long cntr = val.cntr;
-
- if (minActive == null)
- minActive = cntr;
- else if (cntr < minActive)
- minActive = cntr;
- }
-
- return minActive;
- }
-
- static class QueryCounter extends AtomicInteger {
- public QueryCounter(int initialValue) {
- super(initialValue);
- }
-
- boolean increment2() {
- for (;;) {
- int current = get();
- int next = current + 1;
-
- if (current == 0)
- return false;
-
- if (compareAndSet(current, next))
- return true;
- }
- }
- }
-
- private ReadWriteLock rwLock = new ReentrantReadWriteLock();
-
- MvccQueryVersion queryVersion() {
- rwLock.readLock().lock();
-
- long useCntr = commitCntr.get();
-
- Set<TxId> txs = new HashSet<>();
-
- Long minActive = minActive(txs);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive;
-
- MvccQueryVersion qryVer = new MvccQueryVersion(new CoordinatorCounter(useCntr), txs);
-
- for (;;) {
- QueryCounter qryCnt = activeQueries.get(useCntr);
-
- if (qryCnt != null) {
- boolean inc = qryCnt.increment2();
-
- if (!inc) {
- activeQueries.remove(useCntr, qryCnt);
-
- continue;
- }
- }
- else {
- qryCnt = new QueryCounter(1);
-
- if (activeQueries.putIfAbsent(useCntr, qryCnt) != null)
- continue;
- }
-
- break;
- }
-
- rwLock.readLock().unlock();
-
- return qryVer;
- }
-
- void queryDone(CoordinatorCounter cntr) {
- AtomicInteger qryCnt = activeQueries.get(cntr.cntr);
-
- assert qryCnt != null : cntr.cntr;
-
- int left = qryCnt.decrementAndGet();
-
- assert left >= 0 : left;
-
- if (left == 0)
- activeQueries.remove(cntr.cntr, qryCnt);
- }
-
- CoordinatorCounter cleanupVersion() {
- rwLock.writeLock().lock();
-
- long useCntr = commitCntr.get();
-
- Long minActive = minActive(null);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- for (Long qryCntr : activeQueries.keySet()) {
- if (qryCntr <= useCntr)
- useCntr = qryCntr - 1;
- }
-
- rwLock.writeLock().unlock();
-
- return new CoordinatorCounter(useCntr);
- }
-
- @Override public String toString() {
- return S.toString(Coordinator.class, this);
- }
- }
-
- /**
- *
- */
- static class CoordinatorCounter implements Comparable<CoordinatorCounter> {
- /** */
- private final long topVer; // TODO
-
- /** */
- private final long cntr;
-
- CoordinatorCounter(long cntr) {
- this.topVer = 1;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(CoordinatorCounter o) {
- return Long.compare(cntr, o.cntr);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- CoordinatorCounter that = (CoordinatorCounter)o;
-
- return cntr == that.cntr;
- }
-
- @Override public int hashCode() {
- return (int)(cntr ^ (cntr >>> 32));
- }
-
- @Override public String toString() {
- return "Cntr [c=" + cntr + ']';
- }
- }
-
- /**
- *
- */
- static class MvccUpdateVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final TxId txId;
-
- /**
- * @param cntr
- */
- MvccUpdateVersion(CoordinatorCounter cntr, TxId txId) {
- assert cntr != null;
-
- this.cntr = cntr;
- this.txId = txId;
- }
-
- @Override public String toString() {
- return S.toString(MvccUpdateVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccQueryVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- MvccQueryVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
-
- @Override public String toString() {
- return S.toString(MvccQueryVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class TxId {
- long cntr = -1;
-
- /** */
- @GridToStringInclude
- final long id;
-
- TxId(long id) {
- this.id = id;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TxId txId = (TxId) o;
-
- return id == txId.id;
- }
-
- @Override public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override public String toString() {
- return S.toString(TxId.class, this);
- }
- }
-
- /**
- *
- */
- static class SqlKey implements Comparable<SqlKey> {
- /** */
- final Comparable key;
-
- /** */
- final Comparable val;
-
- /** */
- final CoordinatorCounter cntr;
-
- public SqlKey(Object key, Object val, CoordinatorCounter cntr) {
- this.key = (Comparable)key;
- this.val = (Comparable)val;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(@NotNull SqlKey o) {
- int cmp;
-
- if (val != null && o.val != null)
- cmp = val.compareTo(o.val);
- else {
- if (val != null)
- cmp = 1;
- else
- cmp = o.val == null ? 0 : -1;
- }
-
-
- if (cmp == 0) {
- cmp = key.compareTo(o.key);
-
- if (cmp == 0)
- cmp = cntr.compareTo(o.cntr);
- }
-
- return cmp;
- }
-
- @Override public String toString() {
- return "SqlKey [key=" + key + ", val=" + val + ']';
- }
- }
-
- /**
- *
- */
- static class DataStore {
- /** */
- private final ConcurrentHashMap<Object, ReentrantLock> locks = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, MvccValue> mainIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, List<MvccValue>> mvccIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentSkipListMap<SqlKey, MvccSqlValue> mvccSqlIdx = new ConcurrentSkipListMap<>();
-
- void cleanup(CoordinatorCounter cleanupCntr) {
- for (Map.Entry<Object, List<MvccValue>> e : mvccIdx.entrySet()) {
- lockEntry(e.getKey());
-
- try {
- List<MvccValue> list = e.getValue();
-
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (val.ver.cntr.compareTo(cleanupCntr) <= 0) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6_1("cleanup",
- e.getKey(), val.val, val.ver, cleanupCntr.cntr, null, null));
- }
-
- MvccValue prev;
-
- if (val.val != null)
- prev = mainIdx.put(e.getKey(), val);
- else
- prev = mainIdx.remove(e.getKey());
-
- if (prev != null) {
- SqlKey key = new SqlKey(e.getKey(), prev.val, prev.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
-
- for (int j = 0; j <= i; j++) {
- MvccValue rmvd = list.remove(0);
-
- assert rmvd != null;
-
- if (j != i || rmvd.val == null) {
- SqlKey key = new SqlKey(e.getKey(), rmvd.val, rmvd.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
- }
-
- if (list.isEmpty())
- mvccIdx.remove(e.getKey());
-
- break;
- }
- }
- }
- }
- finally {
- unlockEntry(e.getKey());
- }
- }
- }
-
- void lockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.lock();
- }
-
- void unlockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.unlock();
- }
-
- void updateEntry(Object key, Object val, MvccUpdateVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list == null) {
- Object old = mvccIdx.putIfAbsent(key, list = new ArrayList<>());
-
- assert old == null;
- }
-
- MvccValue prevVal = null;
-
- synchronized (list) {
- if (!list.isEmpty())
- prevVal = list.get(list.size() - 1);
-
- list.add(new MvccValue(val, ver));
- }
-
- if (prevVal == null)
- prevVal = mainIdx.get(key);
-
- if (prevVal != null) {
- SqlKey prevKey = new SqlKey(key, prevVal.val, prevVal.ver.cntr);
-
- MvccSqlValue old =
- mvccSqlIdx.put(prevKey, new MvccSqlValue(prevVal.val, prevVal.ver, ver));
-
- assert old != null;
- }
-
- mvccSqlIdx.put(new SqlKey(key, val, ver.cntr), new MvccSqlValue(val, ver, null));
- }
-
- Object lastValue(Object key) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- if (list.size() > 0)
- return list.get(list.size() - 1).val;
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- return val != null ? val.val : null;
- }
-
- Map<Object, Object> sqlQuery(MvccQueryVersion qryVer) {
- Map<Object, Object> res = new HashMap<>();
-
- for (Map.Entry<SqlKey, MvccSqlValue> e : mvccSqlIdx.entrySet()) {
- MvccSqlValue val = e.getValue();
-
- if (!versionVisible(val.ver, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sql skip mvcc val", e.getKey().key, val.val, val.ver));
- }
-
- continue;
- }
-
- MvccUpdateVersion newVer = val.newVer;
-
- if (newVer != null && versionVisible(newVer, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql skip mvcc val2", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- continue;
- }
-
- Object old = res.put(e.getKey().key, e.getValue().val);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql get mvcc val", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- if (old != null) {
- TestDebugLog.printAllAndExit("Already has value for key [key=" + e.getKey().key +
- ", qryVer=" + qryVer +
- ", oldVal=" + old +
- ", newVal=" + e.getValue().val +
- ']');
- }
-
- assert old == null;
- }
-
- return res;
- }
-
- private boolean versionVisible(MvccUpdateVersion ver, MvccQueryVersion qryVer) {
- int cmp = ver.cntr.compareTo(qryVer.cntr);
-
- return cmp <= 0 && !qryVer.activeTxs.contains(ver.txId);
- }
-
- Object get(Object key, MvccQueryVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (!versionVisible(val.ver, ver))
- continue;
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read mvcc val", key, val, val.ver));
- }
-
- return val.val;
- }
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- if (val != null) {
- int cmp = val.ver.cntr.compareTo(ver.cntr);
-
- assert cmp <= 0 : "Committed [ver=" + val.ver + ", qryVer=" + ver.cntr + ']';
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted val", key, val, val.ver));
- }
- else {
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted null", key, null, null));
- }
-
- return val != null ? val.val : null;
- }
-
- private ReentrantLock lock(Object key) {
- ReentrantLock e = locks.get(key);
-
- if (e == null) {
- ReentrantLock old = locks.putIfAbsent(key, e = new ReentrantLock());
-
- if (old != null)
- e = old;
- }
-
- return e;
- }
- }
-
- /**
- *
- */
- static class MvccValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- MvccValue(Object val, MvccUpdateVersion ver) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- }
-
- @Override public String toString() {
- return S.toString(MvccValue.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccSqlValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion newVer;
-
- MvccSqlValue(Object val, MvccUpdateVersion ver, MvccUpdateVersion newVer) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- this.newVer = newVer;
- }
-
- @Override public String toString() {
- return S.toString(MvccSqlValue.class, this);
- }
- }
-
- static void log(String msg) {
- System.out.println(Thread.currentThread() + ": " + msg);
- }
-}
-
-class TestDebugLog {
- /** */
- static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(100_000));
-
- /** */
- private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
-
- static class Message {
- String thread = Thread.currentThread().getName();
-
- String msg;
-
- long ts = U.currentTimeMillis();
-
- public Message(String msg) {
- this.msg = msg;
- }
-
- public String toString() {
- return "Msg [msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg2 extends Message{
- Object v1;
- Object v2;
-
- public Msg2(String msg, Object v1, Object v2) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", msg=" + msg +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg3 extends Message{
- Object v1;
- Object v2;
- Object v3;
-
- public Msg3(String msg, Object v1, Object v2, Object v3) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg4 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
-
- public Msg4(String msg, Object v1, Object v2, Object v3, Object v4) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", v4=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg6 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", txId=" + v1 +
- ", id1=" + v2 +
- ", v1=" + v3 +
- ", id2=" + v4 +
- ", v2=" + v5 +
- ", cntr=" + v6 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
- static class Msg6_1 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6_1(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", key=" + v1 +
- ", val=" + v2 +
- ", ver=" + v3 +
- ", cleanupC=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class EntryMessage extends Message {
- Object key;
- Object val;
-
- public EntryMessage(Object key, Object val, String msg) {
- super(msg);
-
- this.key = key;
- this.val = val;
- }
-
- public String toString() {
- return "EntryMsg [key=" + key + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class PartMessage extends Message {
- int p;
- Object val;
-
- public PartMessage(int p, Object val, String msg) {
- super(msg);
-
- this.p = p;
- this.val = val;
- }
-
- public String toString() {
- return "PartMessage [p=" + p + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static final boolean out = false;
-
- public static void addMessage(String msg) {
- msgs.add(new Message(msg));
-
- if (out)
- System.out.println(msg);
- }
-
- public static void addEntryMessage(Object key, Object val, String msg) {
- if (key instanceof KeyCacheObject)
- key = ((KeyCacheObject)key).value(null, false);
-
- EntryMessage msg0 = new EntryMessage(key, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- public static void addPartMessage(int p, Object val, String msg) {
- PartMessage msg0 = new PartMessage(p, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- static void printAllAndExit(String msg) {
- System.out.println(msg);
-
- TestDebugLog.addMessage(msg);
-
- List<Object> msgs = TestDebugLog.printMessages(true, null);
-
- TestDebugLog.printMessages0(msgs, "test_debug_update.txt");
-
- TestDebugLog.printMessagesForThread(msgs, Thread.currentThread().getName(), "test_debug_thread.txt");
-
- System.exit(1);
- }
-
- public static void printMessagesForThread(List<Object> msgs0, String thread0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String thread = ((Message) msg).thread;
-
- if (thread.equals(thread0))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static void printMessages0(List<Object> msgs0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String msg0 = ((Message) msg).msg;
-
- if (msg0.equals("tx done") || msg0.equals("update") || msg0.equals("cleanup"))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static List<Object> printMessages(boolean file, Integer part) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (part != null && msg instanceof PartMessage) {
- if (((PartMessage) msg).p != part)
- continue;
- }
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0)
- System.out.println(msg);
- }
-
- return msgs0;
- }
-
- public static void printKeyMessages(boolean file, Object key) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- System.out.println(msg);
- }
- }
- }
-
- public static void clear() {
- msgs.clear();
- }
-
- public static void clearEntries() {
- for (Iterator it = msgs.iterator(); it.hasNext();) {
- Object msg = it.next();
-
- if (msg instanceof EntryMessage)
- it.remove();
- }
- }
-
-}
\ No newline at end of file
[18/35] ignite git commit: ignite-6181 Tx rollback on timeout
Posted by sb...@apache.org.
ignite-6181 Tx rollback on timeout
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5af30cf1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5af30cf1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5af30cf1
Branch: refs/heads/ignite-3479
Commit: 5af30cf118aeb3910398e6b15dbe2a51b62746d7
Parents: 27295f2
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 11:20:16 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 11:20:16 2017 +0300
----------------------------------------------------------------------
.../IgniteDiagnosticPrepareContext.java | 4 +-
.../processors/cache/CacheMetricsImpl.java | 2 +-
.../processors/cache/GridCacheAdapter.java | 18 +-
.../processors/cache/GridCacheMapEntry.java | 9 +-
.../processors/cache/GridCacheMvccManager.java | 5 +-
.../GridDistributedTxRemoteAdapter.java | 2 +-
.../distributed/dht/GridDhtTxFinishFuture.java | 2 +-
.../cache/distributed/dht/GridDhtTxLocal.java | 6 +-
.../distributed/dht/GridDhtTxLocalAdapter.java | 4 +-
.../colocated/GridDhtColocatedLockFuture.java | 79 ++-
.../distributed/near/GridNearLockFuture.java | 116 ++--
.../near/GridNearOptimisticTxPrepareFuture.java | 3 +-
.../near/GridNearTransactionalCache.java | 3 -
.../near/GridNearTxFastFinishFuture.java | 82 +++
.../near/GridNearTxFinishFuture.java | 23 +-
.../cache/distributed/near/GridNearTxLocal.java | 303 ++++++---
.../distributed/near/NearTxFinishFuture.java | 31 +
.../cache/transactions/IgniteInternalTx.java | 2 +-
.../cache/transactions/IgniteTxAdapter.java | 13 +-
.../cache/transactions/IgniteTxHandler.java | 26 +-
.../transactions/IgniteTxLocalAdapter.java | 10 +-
.../cache/transactions/IgniteTxLocalEx.java | 7 +-
.../cache/transactions/IgniteTxManager.java | 80 +--
.../timeout/GridTimeoutProcessor.java | 23 +-
.../processors/cache/CacheTxFastFinishTest.java | 9 +-
.../cache/IgniteTxConfigCacheSelfTest.java | 14 +
.../IgniteCacheThreadLocalTxTest.java | 223 +++++++
.../IgniteOptimisticTxSuspendResumeTest.java | 6 +-
...ionedMultiNodeLongTxTimeout2FullApiTest.java | 34 +
.../TxRollbackOnTimeoutNearCacheTest.java | 28 +
...ollbackOnTimeoutNoDeadlockDetectionTest.java | 47 ++
.../transactions/TxRollbackOnTimeoutTest.java | 655 +++++++++++++++++++
.../IgniteCacheFullApiSelfTestSuite.java | 2 +
.../testsuites/IgniteCacheTestSuite6.java | 10 +
.../hadoop/impl/HadoopTxConfigCacheTest.java | 4 +-
35 files changed, 1627 insertions(+), 258 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java
index 378dc74..ed8d35e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteDiagnosticPrepareContext.java
@@ -75,7 +75,7 @@ public class IgniteDiagnosticPrepareContext {
* @param keys Entry keys.
* @param msg Initial message.
*/
- public void txKeyInfo(UUID nodeId, int cacheId, Collection<KeyCacheObject> keys, String msg) {
+ public void txKeyInfo(UUID nodeId, int cacheId, Collection<KeyCacheObject> keys, String msg) {
closure(nodeId).add(msg, new TxEntriesInfoClosure(cacheId, keys));
}
@@ -280,4 +280,4 @@ public class IgniteDiagnosticPrepareContext {
}
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
index d03a6f8..413b60d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheMetricsImpl.java
@@ -328,7 +328,7 @@ public class CacheMetricsImpl implements CacheMetrics {
/** {@inheritDoc} */
@Override public int getTxDhtThreadMapSize() {
- return cctx.isNear() && dhtCtx != null ? dhtCtx.tm().threadMapSize() : -1;
+ return cctx.tm().threadMapSize();
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 92a8245..32b1b99 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -78,6 +78,7 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.IgniteTransactionsEx;
import org.apache.ignite.internal.IgnitionEx;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.cluster.ClusterTopologyServerNotFoundException;
import org.apache.ignite.internal.cluster.IgniteClusterEx;
@@ -1857,7 +1858,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return new GridFinishedFuture<>(e);
}
- tx = ctx.tm().threadLocalTx(ctx.systemTx() ? ctx : null);
+ tx = ctx.tm().threadLocalTx(ctx);
}
if (tx == null || tx.implicit()) {
@@ -4057,7 +4058,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
return t;
}
- catch (IgniteInterruptedCheckedException | IgniteTxHeuristicCheckedException e) {
+ catch (IgniteInterruptedCheckedException | IgniteTxHeuristicCheckedException | NodeStoppingException e) {
throw e;
}
catch (IgniteCheckedException e) {
@@ -4071,7 +4072,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
catch (IgniteCheckedException | AssertionError | RuntimeException e1) {
U.error(log, "Failed to rollback transaction (cache may contain stale locks): " + tx, e1);
- e.addSuppressed(e1);
+ if (e != e1)
+ e.addSuppressed(e1);
}
}
@@ -4202,7 +4204,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
try {
return tFut.get();
}
- catch (IgniteTxRollbackCheckedException e) {
+ catch (IgniteTxRollbackCheckedException | NodeStoppingException e) {
throw e;
}
catch (IgniteCheckedException e1) {
@@ -4210,7 +4212,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
tx0.rollbackNearTxLocalAsync();
}
catch (Throwable e2) {
- e1.addSuppressed(e2);
+ if (e1 != e2)
+ e1.addSuppressed(e2);
}
throw e1;
@@ -4241,7 +4244,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
try {
return tFut.get();
}
- catch (IgniteTxRollbackCheckedException e) {
+ catch (IgniteTxRollbackCheckedException | NodeStoppingException e) {
throw e;
}
catch (IgniteCheckedException e1) {
@@ -4249,7 +4252,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
tx0.rollbackNearTxLocalAsync();
}
catch (Throwable e2) {
- e1.addSuppressed(e2);
+ if (e2 != e1)
+ e1.addSuppressed(e2);
}
throw e1;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 958f156..7b60b9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -2463,7 +2463,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
/** {@inheritDoc} */
@Nullable @Override public CacheObject peek(@Nullable IgniteCacheExpiryPolicy plc)
throws GridCacheEntryRemovedException, IgniteCheckedException {
- IgniteInternalTx tx = cctx.tm().localTxx();
+ IgniteInternalTx tx = cctx.tm().localTx();
AffinityTopologyVersion topVer = tx != null ? tx.topologyVersion() : cctx.affinity().affinityTopologyVersion();
@@ -3110,10 +3110,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
* @return Current transaction.
*/
private IgniteTxLocalAdapter currentTx() {
- if (cctx.isDht())
- return cctx.dht().near().context().tm().localTx();
- else
- return cctx.tm().localTx();
+ return cctx.tm().localTx();
}
/** {@inheritDoc} */
@@ -3491,7 +3488,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
- IgniteInternalTx tx = cctx.tm().localTxx();
+ IgniteInternalTx tx = cctx.tm().localTx();
if (tx != null) {
IgniteTxEntry e = tx.entry(txKey());
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
index 71df71f..a7d9ce7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMvccManager.java
@@ -478,13 +478,14 @@ public class GridCacheMvccManager extends GridCacheSharedManagerAdapter {
/**
* @param fut Future.
* @param futId Future ID.
+ * @return {@code True} if added.
*/
- public void addFuture(final GridCacheFuture<?> fut, final IgniteUuid futId) {
+ public boolean addFuture(final GridCacheFuture<?> fut, final IgniteUuid futId) {
GridCacheFuture<?> old = futs.put(futId, fut);
assert old == null : old;
- onFutureAdded(fut);
+ return onFutureAdded(fut);
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
index ea6461d..e5bcc46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/GridDistributedTxRemoteAdapter.java
@@ -855,7 +855,7 @@ public abstract class GridDistributedTxRemoteAdapter extends IgniteTxAdapter
// Note that we don't evict near entries here -
// they will be deleted by their corresponding transactions.
if (state(ROLLING_BACK) || state() == UNKNOWN) {
- cctx.tm().rollbackTx(this);
+ cctx.tm().rollbackTx(this, false);
state(ROLLED_BACK);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
index 5311ddc..6380710 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxFinishFuture.java
@@ -227,7 +227,7 @@ public final class GridDhtTxFinishFuture<K, V> extends GridCacheCompoundIdentity
try {
boolean nodeStop = err != null && X.hasCause(err, NodeStoppingException.class);
- this.tx.tmFinish(err == null, nodeStop);
+ this.tx.tmFinish(err == null, nodeStop, false);
}
catch (IgniteCheckedException finishErr) {
U.error(log, "Failed to finish tx: " + tx, e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
index ab5631e..28cc018 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocal.java
@@ -436,7 +436,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
if (prepFut != null)
prepFut.get(); // Check for errors.
- boolean finished = localFinish(commit);
+ boolean finished = localFinish(commit, false);
if (!finished)
err = new IgniteCheckedException("Failed to finish transaction [commit=" + commit +
@@ -542,7 +542,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
/** {@inheritDoc} */
@SuppressWarnings({"CatchGenericClass", "ThrowableInstanceNeverThrown"})
- @Override public boolean localFinish(boolean commit) throws IgniteCheckedException {
+ @Override public boolean localFinish(boolean commit, boolean clearThreadMap) throws IgniteCheckedException {
assert nearFinFutId != null || isInvalidate() || !commit || isSystemInvalidate()
|| onePhaseCommit() || state() == PREPARED :
"Invalid state [nearFinFutId=" + nearFinFutId + ", isInvalidate=" + isInvalidate() + ", commit=" + commit +
@@ -550,7 +550,7 @@ public class GridDhtTxLocal extends GridDhtTxLocalAdapter implements GridCacheMa
assert nearMiniId != 0;
- return super.localFinish(commit);
+ return super.localFinish(commit, clearThreadMap);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
index 86eac42..e4a7141 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxLocalAdapter.java
@@ -734,7 +734,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
/** {@inheritDoc} */
@SuppressWarnings({"CatchGenericClass", "ThrowableInstanceNeverThrown"})
- @Override public boolean localFinish(boolean commit) throws IgniteCheckedException {
+ @Override public boolean localFinish(boolean commit, boolean clearThreadMap) throws IgniteCheckedException {
if (log.isDebugEnabled())
log.debug("Finishing dht local tx [tx=" + this + ", commit=" + commit + "]");
@@ -773,7 +773,7 @@ public abstract class GridDhtTxLocalAdapter extends IgniteTxLocalAdapter {
if (commit && !isRollbackOnly())
userCommit();
else
- userRollback();
+ userRollback(clearThreadMap);
}
catch (IgniteCheckedException e) {
err = e;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
index 58c6319..e4f4601 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedLockFuture.java
@@ -26,6 +26,7 @@ import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
@@ -54,6 +55,7 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLock
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
@@ -99,6 +101,10 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
/** Logger. */
private static IgniteLogger msgLog;
+ /** Done field updater. */
+ private static final AtomicIntegerFieldUpdater<GridDhtColocatedLockFuture> DONE_UPD =
+ AtomicIntegerFieldUpdater.newUpdater(GridDhtColocatedLockFuture.class, "done");
+
/** Cache registry. */
@GridToStringExclude
private final GridCacheContext<?, ?> cctx;
@@ -146,6 +152,10 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
/** Map of current values. */
private final Map<KeyCacheObject, IgniteBiTuple<GridCacheVersion, CacheObject>> valMap;
+ /** */
+ @SuppressWarnings("UnusedDeclaration")
+ private volatile int done;
+
/** Trackable flag (here may be non-volatile). */
private boolean trackable;
@@ -226,12 +236,6 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
log = U.logger(cctx.kernalContext(), logRef, GridDhtColocatedLockFuture.class);
}
- if (timeout > 0) {
- timeoutObj = new LockTimeoutObject();
-
- cctx.time().addTimeoutObject(timeoutObj);
- }
-
valMap = new ConcurrentHashMap8<>();
}
@@ -322,6 +326,8 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
else {
IgniteTxEntry txEntry = tx.entry(txKey);
+ assert txEntry != null;
+
txEntry.cached(entry);
// Check transaction entries (corresponding tx entries must be enlisted in transaction).
@@ -332,7 +338,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
threadId,
lockVer,
true,
- tx.entry(txKey).locked(),
+ txEntry.locked(),
inTx(),
inTx() && tx.implicitSingle(),
false,
@@ -399,7 +405,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
* @param success Success flag.
*/
public void complete(boolean success) {
- onComplete(success, true);
+ onComplete(success, true, true);
}
/**
@@ -533,7 +539,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
/** {@inheritDoc} */
@Override public boolean cancel() {
if (onCancelled())
- onComplete(false, true);
+ onComplete(false, true, true);
return isCancelled();
}
@@ -556,7 +562,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
if (err != null)
success = false;
- return onComplete(success, true);
+ return onComplete(success, true, true);
}
/**
@@ -564,19 +570,32 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
*
* @param success {@code True} if lock was acquired.
* @param distribute {@code True} if need to distribute lock removal in case of failure.
+ * @param restoreTimeout {@code True} if need restore tx timeout callback.
* @return {@code True} if complete by this operation.
*/
- private boolean onComplete(boolean success, boolean distribute) {
- if (log.isDebugEnabled())
+ private boolean onComplete(boolean success, boolean distribute, boolean restoreTimeout) {
+ if (log.isDebugEnabled()) {
log.debug("Received onComplete(..) callback [success=" + success + ", distribute=" + distribute +
", fut=" + this + ']');
+ }
+
+ if (!DONE_UPD.compareAndSet(this, 0, 1))
+ return false;
if (!success)
undoLocks(distribute, true);
- if (tx != null)
+ if (tx != null) {
cctx.tm().txContext(tx);
+ if (restoreTimeout && tx.trackTimeout()) {
+ // Need restore timeout before onDone is called and next tx operation can proceed.
+ boolean add = tx.addTimeoutHandler();
+
+ assert add;
+ }
+ }
+
if (super.onDone(success, err)) {
if (log.isDebugEnabled())
log.debug("Completing future: " + this);
@@ -675,6 +694,30 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
* part. Note that if primary node leaves grid, the future will fail and transaction will be rolled back.
*/
void map() {
+ if (tx != null && tx.trackTimeout()) {
+ if (!tx.removeTimeoutHandler()) {
+ tx.finishFuture().listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
+ @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut) {
+ IgniteTxTimeoutCheckedException err = new IgniteTxTimeoutCheckedException("Failed to " +
+ "acquire lock, transaction was rolled back on timeout [timeout=" + tx.timeout() +
+ ", tx=" + tx + ']');
+
+ onError(err);
+
+ onComplete(false, false, false);
+ }
+ });
+
+ return;
+ }
+ }
+
+ if (timeout > 0) {
+ timeoutObj = new LockTimeoutObject();
+
+ cctx.time().addTimeoutObject(timeoutObj);
+ }
+
// Obtain the topology version to use.
AffinityTopologyVersion topVer = cctx.mvcc().lastExplicitLockTopologyVersion(threadId);
@@ -930,7 +973,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
if (log.isDebugEnabled())
log.debug("Entry being locked did not pass filter (will not lock): " + entry);
- onComplete(false, false);
+ onComplete(false, false, true);
return;
}
@@ -1307,7 +1350,7 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
if (log.isDebugEnabled())
log.debug("Entry being locked did not pass filter (will not lock): " + entry);
- onComplete(false, false);
+ onComplete(false, false, true);
return false;
}
@@ -1419,12 +1462,12 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
U.warn(log, "Failed to detect deadlock.", e);
}
- onComplete(false, true);
+ onComplete(false, true, true);
}
});
}
else
- onComplete(false, true);
+ onComplete(false, true, true);
}
/** {@inheritDoc} */
@@ -1673,4 +1716,4 @@ public final class GridDhtColocatedLockFuture extends GridCacheCompoundIdentityF
return S.toString(MiniFuture.class, this, "node", node.id(), "super", super.toString());
}
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
index c947715..3d9989d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearLockFuture.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
@@ -50,6 +51,7 @@ import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCa
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTopologyFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTransactionalCacheAdapter;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.TxDeadlock;
@@ -89,6 +91,10 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
/** Logger reference. */
private static final AtomicReference<IgniteLogger> logRef = new AtomicReference<>();
+ /** Done field updater. */
+ private static final AtomicIntegerFieldUpdater<GridNearLockFuture> DONE_UPD =
+ AtomicIntegerFieldUpdater.newUpdater(GridNearLockFuture.class, "done");
+
/** */
private static IgniteLogger log;
@@ -142,8 +148,9 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
/** Map of current values. */
private final Map<KeyCacheObject, IgniteBiTuple<GridCacheVersion, CacheObject>> valMap;
- /** Trackable flag. */
- private boolean trackable = true;
+ /** */
+ @SuppressWarnings("UnusedDeclaration")
+ private volatile int done;
/** Keys locked so far. */
@SuppressWarnings({"FieldAccessedSynchronizedAndUnsynchronized"})
@@ -184,6 +191,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
* @param filter Filter.
* @param skipStore skipStore
* @param keepBinary Keep binary flag.
+ * @param recovery Recovery flag.
*/
public GridNearLockFuture(
GridCacheContext<?, ?> cctx,
@@ -230,12 +238,6 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
if (log == null)
log = U.logger(cctx.kernalContext(), logRef, GridNearLockFuture.class);
- if (timeout > 0) {
- timeoutObj = new LockTimeoutObject();
-
- cctx.time().addTimeoutObject(timeoutObj);
- }
-
valMap = new ConcurrentHashMap8<>();
}
@@ -260,12 +262,12 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
/** {@inheritDoc} */
@Override public boolean trackable() {
- return trackable;
+ return true;
}
/** {@inheritDoc} */
@Override public void markNotTrackable() {
- trackable = false;
+ // No-op.
}
/**
@@ -434,7 +436,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
* @param success Success flag.
*/
public void complete(boolean success) {
- onComplete(success, true);
+ onComplete(success, true, true);
}
/**
@@ -655,7 +657,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
log.debug("Local lock acquired for entries [fut=" + this + ", entries=" + entries + "]");
}
- onComplete(true, true);
+ onComplete(true, true, true);
return true;
}
@@ -666,7 +668,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
/** {@inheritDoc} */
@Override public boolean cancel() {
if (onCancelled())
- onComplete(false, true);
+ onComplete(false, true, true);
return isCancelled();
}
@@ -690,7 +692,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
if (err != null)
success = false;
- return onComplete(success, true);
+ return onComplete(success, true, true);
}
/**
@@ -698,19 +700,32 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
*
* @param success {@code True} if lock was acquired.
* @param distribute {@code True} if need to distribute lock removal in case of failure.
+ * @param restoreTimeout {@code True} if need restore tx timeout callback.
* @return {@code True} if complete by this operation.
*/
- private boolean onComplete(boolean success, boolean distribute) {
- if (log.isDebugEnabled())
+ private boolean onComplete(boolean success, boolean distribute, boolean restoreTimeout) {
+ if (log.isDebugEnabled()) {
log.debug("Received onComplete(..) callback [success=" + success + ", distribute=" + distribute +
", fut=" + this + ']');
+ }
+
+ if (!DONE_UPD.compareAndSet(this, 0, 1))
+ return false;
if (!success)
undoLocks(distribute, true);
- if (tx != null)
+ if (tx != null) {
cctx.tm().txContext(tx);
+ if (restoreTimeout && tx.trackTimeout()) {
+ // Need restore timeout before onDone is called and next tx operation can proceed.
+ boolean add = tx.addTimeoutHandler();
+
+ assert add;
+ }
+ }
+
if (super.onDone(success, err)) {
if (log.isDebugEnabled())
log.debug("Completing future: " + this);
@@ -770,6 +785,34 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
* part. Note that if primary node leaves grid, the future will fail and transaction will be rolled back.
*/
void map() {
+ if (tx != null && tx.trackTimeout()) {
+ if (!tx.removeTimeoutHandler()) {
+ tx.finishFuture().listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
+ @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut) {
+ IgniteTxTimeoutCheckedException err = new IgniteTxTimeoutCheckedException("Failed to " +
+ "acquire lock, transaction was rolled back on timeout [timeout=" + tx.timeout() +
+ ", tx=" + tx + ']');
+
+ onError(err);
+
+ onComplete(false, false, false);
+ }
+ });
+
+ return;
+ }
+ }
+
+ if (timeout > 0) {
+ timeoutObj = new LockTimeoutObject();
+
+ cctx.time().addTimeoutObject(timeoutObj);
+ }
+
+ boolean added = cctx.mvcc().addFuture(this);
+
+ assert added : this;
+
// Obtain the topology version to use.
long threadId = Thread.currentThread().getId();
@@ -971,19 +1014,13 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
GridNearCacheEntry entry = null;
try {
- entry = cctx.near().entryExx(
- key,
- topVer);
+ entry = cctx.near().entryExx(key, topVer);
- if (!cctx.isAll(
- entry,
- filter)) {
+ if (!cctx.isAll(entry, filter)) {
if (log.isDebugEnabled())
log.debug("Entry being locked did not pass filter (will not lock): " + entry);
- onComplete(
- false,
- false);
+ onComplete(false, false, true);
return;
}
@@ -1004,10 +1041,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
if (cand != null) {
if (tx == null && !cand.reentry())
- cctx.mvcc().addExplicitLock(
- threadId,
- cand,
- topVer);
+ cctx.mvcc().addExplicitLock(threadId,cand,topVer);
IgniteBiTuple<GridCacheVersion, CacheObject> val = entry.versionedValue();
@@ -1032,9 +1066,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
if (val != null) {
dhtVer = val.get1();
- valMap.put(
- key,
- val);
+ valMap.put(key, val);
}
if (!cand.reentry()) {
@@ -1083,9 +1115,7 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
distributedKeys.add(key);
if (tx != null)
- tx.addKeyMapping(
- txKey,
- mapping.node());
+ tx.addKeyMapping(txKey, mapping.node());
req.addKeyBytes(
key,
@@ -1098,14 +1128,16 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
if (cand.reentry())
explicit = tx != null && !entry.hasLockCandidate(tx.xidVersion());
}
- else
+ else {
+ if (timedOut)
+ return;
+
// Ignore reentries within transactions.
explicit = tx != null && !entry.hasLockCandidate(tx.xidVersion());
+ }
if (explicit)
- tx.addKeyMapping(
- txKey,
- mapping.node());
+ tx.addKeyMapping(txKey, mapping.node());
break;
}
@@ -1476,12 +1508,12 @@ public final class GridNearLockFuture extends GridCacheCompoundIdentityFuture<Bo
U.warn(log, "Failed to detect deadlock.", e);
}
- onComplete(false, true);
+ onComplete(false, true, true);
}
});
}
else
- onComplete(false, true);
+ onComplete(false, true, true);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
index 77c68bd..6d7a862 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearOptimisticTxPrepareFuture.java
@@ -913,8 +913,7 @@ public class GridNearOptimisticTxPrepareFuture extends GridNearOptimisticTxPrepa
return;
if (RCV_RES_UPD.compareAndSet(this, 0, 1)) {
- if (parent.cctx.tm().deadlockDetectionEnabled() &&
- (parent.tx.remainingTime() == -1 || res.error() instanceof IgniteTxTimeoutCheckedException)) {
+ if (parent.tx.remainingTime() == -1 || res.error() instanceof IgniteTxTimeoutCheckedException) {
parent.onTimeout();
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
index 9d0b186..a13da3d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java
@@ -467,9 +467,6 @@ public class GridNearTransactionalCache<K, V> extends GridNearCacheAdapter<K, V>
opCtx != null && opCtx.isKeepBinary(),
opCtx != null && opCtx.recovery());
- if (!ctx.mvcc().addFuture(fut))
- throw new IllegalStateException("Duplicate future ID: " + fut);
-
fut.map();
return fut;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
new file mode 100644
index 0000000..7222697
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+
+import static org.apache.ignite.transactions.TransactionState.COMMITTED;
+import static org.apache.ignite.transactions.TransactionState.COMMITTING;
+import static org.apache.ignite.transactions.TransactionState.PREPARED;
+import static org.apache.ignite.transactions.TransactionState.PREPARING;
+import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
+import static org.apache.ignite.transactions.TransactionState.ROLLING_BACK;
+
+/**
+ *
+ */
+public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternalTx> implements NearTxFinishFuture {
+ /** */
+ private final GridNearTxLocal tx;
+
+ /** */
+ private final boolean commit;
+
+ /**
+ * @param tx Transaction.
+ * @param commit Commit flag.
+ */
+ GridNearTxFastFinishFuture(GridNearTxLocal tx, boolean commit) {
+ this.tx = tx;
+ this.commit = commit;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean commit() {
+ return commit;
+ }
+
+ /**
+ *
+ */
+ public void finish() {
+ try {
+ if (commit) {
+ tx.state(PREPARING);
+ tx.state(PREPARED);
+ tx.state(COMMITTING);
+
+ tx.context().tm().fastFinishTx(tx, true);
+
+ tx.state(COMMITTED);
+ }
+ else {
+ tx.state(PREPARING);
+ tx.state(PREPARED);
+ tx.state(ROLLING_BACK);
+
+ tx.context().tm().fastFinishTx(tx, false);
+
+ tx.state(ROLLED_BACK);
+ }
+ }
+ finally {
+ onDone(tx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index c45eb7b..b6a8855 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -67,7 +67,7 @@ import static org.apache.ignite.transactions.TransactionState.UNKNOWN;
*
*/
public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentityFuture<IgniteInternalTx>
- implements GridCacheFuture<IgniteInternalTx> {
+ implements GridCacheFuture<IgniteInternalTx>, NearTxFinishFuture {
/** */
private static final long serialVersionUID = 0L;
@@ -136,6 +136,11 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
}
/** {@inheritDoc} */
+ @Override public boolean commit() {
+ return commit;
+ }
+
+ /** {@inheritDoc} */
@Override public IgniteUuid futureId() {
return futId;
}
@@ -278,6 +283,13 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
}
}
+ /**
+ *
+ */
+ void forceFinish() {
+ super.onDone(tx, null, false);
+ }
+
/** {@inheritDoc} */
@Override public boolean onDone(IgniteInternalTx tx0, Throwable err) {
if (isDone())
@@ -310,7 +322,7 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
err = new TransactionRollbackException("Failed to commit transaction.", err);
try {
- tx.localFinish(err == null);
+ tx.localFinish(err == null, true);
}
catch (IgniteCheckedException e) {
if (err != null)
@@ -327,7 +339,7 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
finishOnePhase(commit);
try {
- tx.tmFinish(commit, nodeStop);
+ tx.tmFinish(commit, nodeStop, true);
}
catch (IgniteCheckedException e) {
U.error(log, "Failed to finish tx: " + tx, e);
@@ -386,9 +398,10 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
* Initializes future.
*
* @param commit Commit flag.
+ * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
*/
@SuppressWarnings("ForLoopReplaceableByForEach")
- void finish(boolean commit) {
+ public void finish(boolean commit, boolean clearThreadMap) {
if (tx.onNeedCheckBackup()) {
assert tx.onePhaseCommit();
@@ -402,7 +415,7 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
}
try {
- if (tx.localFinish(commit) || (!commit && tx.state() == UNKNOWN)) {
+ if (tx.localFinish(commit, clearThreadMap) || (!commit && tx.state() == UNKNOWN)) {
if ((tx.onePhaseCommit() && needFinishOnePhase(commit)) || (!tx.onePhaseCommit() && mappings != null)) {
if (mappings.single()) {
GridDistributedTxMapping mapping = mappings.singleMapping();
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 058a3ff..8b043d8 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -67,6 +67,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.TransactionProxy;
import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
@@ -89,6 +90,7 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiClosure;
import org.apache.ignite.lang.IgniteClosure;
+import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.security.SecurityPermission;
import org.apache.ignite.transactions.TransactionConcurrency;
@@ -107,7 +109,7 @@ import static org.apache.ignite.internal.processors.cache.transactions.IgniteTxE
import static org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry.SER_READ_NOT_EMPTY_VER;
import static org.apache.ignite.transactions.TransactionState.COMMITTED;
import static org.apache.ignite.transactions.TransactionState.COMMITTING;
-import static org.apache.ignite.transactions.TransactionState.PREPARED;
+import static org.apache.ignite.transactions.TransactionState.MARKED_ROLLBACK;
import static org.apache.ignite.transactions.TransactionState.PREPARING;
import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
import static org.apache.ignite.transactions.TransactionState.ROLLING_BACK;
@@ -117,7 +119,7 @@ import static org.apache.ignite.transactions.TransactionState.UNKNOWN;
* Replicated user transaction.
*/
@SuppressWarnings("unchecked")
-public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoCloseable {
+public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeoutObject, AutoCloseable {
/** */
private static final long serialVersionUID = 0L;
@@ -126,12 +128,8 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
AtomicReferenceFieldUpdater.newUpdater(GridNearTxLocal.class, IgniteInternalFuture.class, "prepFut");
/** Prepare future updater. */
- private static final AtomicReferenceFieldUpdater<GridNearTxLocal, GridNearTxFinishFuture> COMMIT_FUT_UPD =
- AtomicReferenceFieldUpdater.newUpdater(GridNearTxLocal.class, GridNearTxFinishFuture.class, "commitFut");
-
- /** Rollback future updater. */
- private static final AtomicReferenceFieldUpdater<GridNearTxLocal, GridNearTxFinishFuture> ROLLBACK_FUT_UPD =
- AtomicReferenceFieldUpdater.newUpdater(GridNearTxLocal.class, GridNearTxFinishFuture.class, "rollbackFut");
+ private static final AtomicReferenceFieldUpdater<GridNearTxLocal, NearTxFinishFuture> FINISH_FUT_UPD =
+ AtomicReferenceFieldUpdater.newUpdater(GridNearTxLocal.class, NearTxFinishFuture.class, "finishFut");
/** DHT mappings. */
private IgniteTxMappings mappings;
@@ -144,12 +142,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
/** Commit future. */
@SuppressWarnings("UnusedDeclaration")
@GridToStringExclude
- private volatile GridNearTxFinishFuture commitFut;
-
- /** Rollback future. */
- @SuppressWarnings("UnusedDeclaration")
- @GridToStringExclude
- private volatile GridNearTxFinishFuture rollbackFut;
+ private volatile NearTxFinishFuture finishFut;
/** True if transaction contains near cache entries mapped to local node. */
private boolean nearLocallyMapped;
@@ -170,6 +163,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
protected boolean transform;
/** */
+ private boolean trackTimeout;
+
+ /** */
@GridToStringExclude
private TransactionProxyImpl proxy;
@@ -229,6 +225,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
mappings = implicitSingle ? new IgniteTxMappingsSingleImpl() : new IgniteTxMappingsImpl();
initResult();
+
+ if (timeout() > 0 && !implicit())
+ trackTimeout = cctx.time().addTimeoutObject(this);
}
/** {@inheritDoc} */
@@ -3044,7 +3043,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
/** {@inheritDoc} */
@SuppressWarnings({"CatchGenericClass", "ThrowableInstanceNeverThrown"})
- @Override public boolean localFinish(boolean commit) throws IgniteCheckedException {
+ @Override public boolean localFinish(boolean commit, boolean clearThreadMap) throws IgniteCheckedException {
if (log.isDebugEnabled())
log.debug("Finishing near local tx [tx=" + this + ", commit=" + commit + "]");
@@ -3080,7 +3079,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
if (commit && !isRollbackOnly())
userCommit();
else
- userRollback();
+ userRollback(clearThreadMap);
}
catch (IgniteCheckedException e) {
err = e;
@@ -3146,6 +3145,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
if (!PREP_FUT_UPD.compareAndSet(this, null, fut))
return prepFut;
+ if (trackTimeout)
+ removeTimeoutHandler();
+
if (timeout == -1) {
fut.onDone(this, timeoutException());
@@ -3178,14 +3180,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
if (awaitLastFuture)
txState().awaitLastFuture(cctx);
- prepareAsync().get();
- }
-
- /**
- * @return Prepare future.
- */
- private IgniteInternalFuture<?> prepareAsync() {
- return prepareNearTxLocal();
+ prepareNearTxLocal().get();
}
/**
@@ -3202,42 +3197,43 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
if (log.isDebugEnabled())
log.debug("Committing near local tx: " + this);
+ NearTxFinishFuture fut = finishFut;
+
+ if (fut != null)
+ return chainFinishFuture(fut, true);
+
if (fastFinish()) {
- state(PREPARING);
- state(PREPARED);
- state(COMMITTING);
+ GridNearTxFastFinishFuture fut0;
- cctx.tm().fastFinishTx(this, true);
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true)))
+ return chainFinishFuture(finishFut, true);
- state(COMMITTED);
+ fut0.finish();
- return new GridFinishedFuture<>((IgniteInternalTx)this);
+ return fut0;
}
- final IgniteInternalFuture<?> prepareFut = prepareNearTxLocal();
+ final GridNearTxFinishFuture fut0;
- GridNearTxFinishFuture fut = commitFut;
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, true)))
+ return chainFinishFuture(finishFut, true);
- if (fut != null ||
- !COMMIT_FUT_UPD.compareAndSet(this, null, fut = new GridNearTxFinishFuture<>(cctx, this, true)))
- return commitFut;
+ cctx.mvcc().addFuture(fut0, fut0.futureId());
- cctx.mvcc().addFuture(fut, fut.futureId());
+ final IgniteInternalFuture<?> prepareFut = prepareNearTxLocal();
prepareFut.listen(new CI1<IgniteInternalFuture<?>>() {
@Override public void apply(IgniteInternalFuture<?> f) {
- GridNearTxFinishFuture fut0 = commitFut;
-
try {
// Make sure that here are no exceptions.
prepareFut.get();
- fut0.finish(true);
+ fut0.finish(true, true);
}
catch (Error | RuntimeException e) {
COMMIT_ERR_UPD.compareAndSet(GridNearTxLocal.this, null, e);
- fut0.finish(false);
+ fut0.finish(false, true);
throw e;
}
@@ -3245,12 +3241,12 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
COMMIT_ERR_UPD.compareAndSet(GridNearTxLocal.this, null, e);
if (!(e instanceof NodeStoppingException))
- fut0.finish(false);
+ fut0.finish(false, true);
}
}
});
- return fut;
+ return fut0;
}
/** {@inheritDoc} */
@@ -3269,30 +3265,42 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
* @return Rollback future.
*/
public IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync() {
+ return rollbackNearTxLocalAsync(false);
+ }
+
+ /**
+ * @param onTimeout {@code True} if rolled back asynchronously on timeout.
+ * @return Rollback future.
+ */
+ private IgniteInternalFuture<IgniteInternalTx> rollbackNearTxLocalAsync(final boolean onTimeout) {
if (log.isDebugEnabled())
log.debug("Rolling back near tx: " + this);
+ if (!onTimeout && trackTimeout)
+ removeTimeoutHandler();
+
+ NearTxFinishFuture fut = finishFut;
+
+ if (fut != null)
+ return chainFinishFuture(finishFut, false);
+
if (fastFinish()) {
- state(PREPARING);
- state(PREPARED);
- state(ROLLING_BACK);
+ GridNearTxFastFinishFuture fut0;
- cctx.tm().fastFinishTx(this, false);
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false)))
+ return chainFinishFuture(finishFut, false);
- state(ROLLED_BACK);
+ fut0.finish();
- return new GridFinishedFuture<>((IgniteInternalTx)this);
+ return fut0;
}
- GridNearTxFinishFuture fut = rollbackFut;
+ final GridNearTxFinishFuture fut0;
- if (fut != null)
- return fut;
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, false)))
+ return chainFinishFuture(finishFut, false);
- if (!ROLLBACK_FUT_UPD.compareAndSet(this, null, fut = new GridNearTxFinishFuture<>(cctx, this, false)))
- return rollbackFut;
-
- cctx.mvcc().addFuture(fut, fut.futureId());
+ cctx.mvcc().addFuture(fut0, fut0.futureId());
IgniteInternalFuture<?> prepFut = this.prepFut;
@@ -3307,7 +3315,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
}
- fut.finish(false);
+ fut0.finish(false, !onTimeout);
}
else {
prepFut.listen(new CI1<IgniteInternalFuture<?>>() {
@@ -3321,14 +3329,12 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
log.debug("Got optimistic tx failure [tx=" + this + ", err=" + e + ']');
}
- GridNearTxFinishFuture fut0 = rollbackFut;
-
- fut0.finish(false);
+ fut0.finish(false, !onTimeout);
}
});
}
- return fut;
+ return fut0;
}
/** {@inheritDoc} */
@@ -3337,6 +3343,64 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
}
/**
+ * @param fut Already started finish future.
+ * @param commit Commit flag.
+ * @return Finish future.
+ */
+ private IgniteInternalFuture<IgniteInternalTx> chainFinishFuture(final NearTxFinishFuture fut, final boolean commit) {
+ assert fut != null;
+
+ if (fut.commit() != commit) {
+ final GridNearTxLocal tx = this;
+
+ if (!commit) {
+ final GridNearTxFinishFuture rollbackFut = new GridNearTxFinishFuture<>(cctx, this, false);
+
+ fut.listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
+ @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut0) {
+ if (FINISH_FUT_UPD.compareAndSet(tx, fut, rollbackFut)) {
+ if (tx.state() == COMMITTED) {
+ if (log.isDebugEnabled())
+ log.debug("Failed to rollback, transaction is already committed: " + tx);
+
+ rollbackFut.forceFinish();
+
+ assert rollbackFut.isDone() : rollbackFut;
+ }
+ else {
+ if (!cctx.mvcc().addFuture(rollbackFut, rollbackFut.futureId()))
+ return;
+
+ rollbackFut.finish(false, true);
+ }
+ }
+ }
+ });
+
+ return rollbackFut;
+ }
+ else {
+ final GridFutureAdapter<IgniteInternalTx> fut0 = new GridFutureAdapter<>();
+
+ fut.listen(new IgniteInClosure<IgniteInternalFuture<IgniteInternalTx>>() {
+ @Override public void apply(IgniteInternalFuture<IgniteInternalTx> fut) {
+ if (timedOut())
+ fut0.onDone(new IgniteTxTimeoutCheckedException("Failed to commit transaction, " +
+ "transaction is concurrently rolled back on timeout: " + tx));
+ else
+ fut0.onDone(new IgniteCheckedException("Failed to commit transaction, " +
+ "transaction is concurrently rolled back: " + tx));
+ }
+ });
+
+ return fut0;
+ }
+ }
+
+ return fut;
+ }
+
+ /**
* @return {@code True} if 'fast finish' path can be used for transaction completion.
*/
private boolean fastFinish() {
@@ -3693,38 +3757,53 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
@Override public void close() throws IgniteCheckedException {
TransactionState state = state();
- if (state != ROLLING_BACK && state != ROLLED_BACK && state != COMMITTING && state != COMMITTED)
- rollback();
+ try {
+ if (state == COMMITTED || state == ROLLED_BACK)
+ return;
- synchronized (this) {
- try {
- while (!done())
- wait();
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ if (trackTimeout)
+ removeTimeoutHandler();
- if (!done())
- throw new IgniteCheckedException("Got interrupted while waiting for transaction to complete: " +
- this, e);
+ if (state != ROLLING_BACK && state != COMMITTING)
+ rollback();
+
+ synchronized (this) {
+ try {
+ while (!done())
+ wait();
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ if (!done())
+ throw new IgniteCheckedException("Got interrupted while waiting for transaction to complete: " +
+ this, e);
+ }
}
}
+ finally {
+ // It is possible tx was rolled back asynchronously on timeout and thread map is not cleared yet.
+ boolean cleanup = state == ROLLED_BACK && timedOut();
- if (accessMap != null) {
- assert optimistic();
+ if (cleanup)
+ cctx.tm().clearThreadMap(this);
- for (Map.Entry<IgniteTxKey, IgniteCacheExpiryPolicy> e : accessMap.entrySet()) {
- if (e.getValue().entries() != null) {
- GridCacheContext cctx0 = cctx.cacheContext(e.getKey().cacheId());
+ if (accessMap != null) {
+ assert optimistic();
- if (cctx0.isNear())
- cctx0.near().dht().sendTtlUpdateRequest(e.getValue());
- else
- cctx0.dht().sendTtlUpdateRequest(e.getValue());
+ for (Map.Entry<IgniteTxKey, IgniteCacheExpiryPolicy> e : accessMap.entrySet()) {
+ if (e.getValue().entries() != null) {
+ GridCacheContext cctx0 = cctx.cacheContext(e.getKey().cacheId());
+
+ if (cctx0.isNear())
+ cctx0.near().dht().sendTtlUpdateRequest(e.getValue());
+ else
+ cctx0.dht().sendTtlUpdateRequest(e.getValue());
+ }
}
- }
- accessMap = null;
+ accessMap = null;
+ }
}
}
@@ -4000,13 +4079,69 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements AutoClosea
/**
* @param threadId new owner of transaction.
- * @throws IgniteCheckedException if method executed not in the middle of resume or suspend.
*/
public void threadId(long threadId) {
this.threadId = threadId;
}
/**
+ * @return {@code True} if need register callback which cancels tx on timeout.
+ */
+ public boolean trackTimeout() {
+ return trackTimeout;
+ }
+
+ /**
+ * Removes timeout handler.
+ *
+ * @return {@code True} if handler was removed.
+ */
+ public boolean removeTimeoutHandler() {
+ assert trackTimeout;
+
+ return cctx.time().removeTimeoutObject(this);
+ }
+
+ /**
+ * @return {@code True} if handler was added.
+ */
+ public boolean addTimeoutHandler() {
+ assert trackTimeout;
+
+ return cctx.time().addTimeoutObject(this);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgniteUuid timeoutId() {
+ return xid();
+ }
+
+ /** {@inheritDoc} */
+ @Override public long endTime() {
+ return startTime() + timeout();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void onTimeout() {
+ if (state(MARKED_ROLLBACK, true) || (state() == MARKED_ROLLBACK)) {
+ if (log.isDebugEnabled())
+ log.debug("Will rollback tx on timeout: " + this);
+
+ cctx.kernalContext().closure().runLocalSafe(new Runnable() {
+ @Override public void run() {
+ // Note: if rollback asynchonously on timeout should not clear thread map
+ // since thread started tx still should be able to see this tx.
+ rollbackNearTxLocalAsync(true);
+ }
+ });
+ }
+ else {
+ if (log.isDebugEnabled())
+ log.debug("Skip rollback tx on timeout: " + this);
+ }
+ }
+
+ /**
* Post-lock closure.
*
* @param <T> Return type.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
new file mode 100644
index 0000000..132c754
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
+
+/**
+ *
+ */
+public interface NearTxFinishFuture extends IgniteInternalFuture<IgniteInternalTx> {
+ /**
+ * @return Commit flag.
+ */
+ boolean commit();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
index 7598003..9e06d9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
@@ -634,4 +634,4 @@ public interface IgniteInternalTx {
* @param e Commit error.
*/
public void commitError(Throwable e);
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
index c447436..b5178b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
@@ -370,6 +370,13 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
consistentIdMapper = new ConsistentIdMapper(cctx.discovery());
}
+ /**
+ * @return Shared cache context.
+ */
+ public GridCacheSharedContext<?, ?> context() {
+ return cctx;
+ }
+
/** {@inheritDoc} */
@Override public boolean localResult() {
assert originatingNodeId() != null;
@@ -987,7 +994,7 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
* @return {@code True} if state changed.
*/
@SuppressWarnings({"TooBroadScope"})
- protected boolean state(TransactionState state, boolean timedOut) {
+ protected final boolean state(TransactionState state, boolean timedOut) {
boolean valid = false;
TransactionState prev;
@@ -1068,7 +1075,9 @@ public abstract class IgniteTxAdapter extends GridMetadataAwareAdapter implement
if (valid) {
this.state = state;
- this.timedOut = timedOut;
+
+ if (timedOut)
+ this.timedOut = true;
if (log.isDebugEnabled())
log.debug("Changed transaction state [prev=" + prev + ", new=" + this.state + ", tx=" + this + ']');
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
index b60bab5..38c877b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java
@@ -957,28 +957,30 @@ public class IgniteTxHandler {
}
}
catch (Throwable e) {
- tx.commitError(e);
+ U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
- tx.systemInvalidate(true);
+ if (tx != null) {
+ tx.commitError(e);
- U.error(log, "Failed completing transaction [commit=" + req.commit() + ", tx=" + tx + ']', e);
+ tx.systemInvalidate(true);
- IgniteInternalFuture<IgniteInternalTx> res = null;
+ try {
+ IgniteInternalFuture<IgniteInternalTx> res = tx.rollbackDhtLocalAsync();
- try {
- res = tx.rollbackDhtLocalAsync();
+ // Only for error logging.
+ res.listen(CU.errorLogger(log));
- // Only for error logging.
- res.listen(CU.errorLogger(log));
- }
- catch (Throwable e1) {
- e.addSuppressed(e1);
+ return res;
+ }
+ catch (Throwable e1) {
+ e.addSuppressed(e1);
+ }
}
if (e instanceof Error)
throw (Error)e;
- return res;
+ return new GridFinishedFuture<>(e);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index 7ab921c..143e5cb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -54,6 +54,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.store.CacheStoreManager;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersionConflictContext;
@@ -898,10 +899,11 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
* Commits transaction to transaction manager. Used for one-phase commit transactions only.
*
* @param commit If {@code true} commits transaction, otherwise rollbacks.
+ * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
* @param nodeStop If {@code true} tx is cancelled on node stop.
* @throws IgniteCheckedException If failed.
*/
- public void tmFinish(boolean commit, boolean nodeStop) throws IgniteCheckedException {
+ public void tmFinish(boolean commit, boolean nodeStop, boolean clearThreadMap) throws IgniteCheckedException {
assert onePhaseCommit();
if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
@@ -910,7 +912,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
if (commit)
cctx.tm().commitTx(this);
else
- cctx.tm().rollbackTx(this);
+ cctx.tm().rollbackTx(this, clearThreadMap);
}
state(commit ? COMMITTED : ROLLED_BACK);
@@ -957,7 +959,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
/** {@inheritDoc} */
- @Override public void userRollback() throws IgniteCheckedException {
+ @Override public void userRollback(boolean clearThreadMap) throws IgniteCheckedException {
TransactionState state = state();
if (state != ROLLING_BACK && state != ROLLED_BACK) {
@@ -975,7 +977,7 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
}
if (DONE_FLAG_UPD.compareAndSet(this, 0, 1)) {
- cctx.tm().rollbackTx(this);
+ cctx.tm().rollbackTx(this, clearThreadMap);
if (!internal()) {
Collection<CacheStoreManager> stores = txState.stores(cctx);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
index 307c348..b61b1a9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalEx.java
@@ -18,6 +18,7 @@
package org.apache.ignite.internal.processors.cache.transactions;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.jetbrains.annotations.Nullable;
@@ -41,16 +42,18 @@ public interface IgniteTxLocalEx extends IgniteInternalTx {
public void userCommit() throws IgniteCheckedException;
/**
+ * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
* @throws IgniteCheckedException If rollback failed.
*/
- public void userRollback() throws IgniteCheckedException;
+ public void userRollback(boolean clearThreadMap) throws IgniteCheckedException;
/**
* Finishes transaction (either commit or rollback).
*
* @param commit {@code True} if commit, {@code false} if rollback.
+ * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
* @return {@code True} if state has been changed.
* @throws IgniteCheckedException If finish failed.
*/
- public boolean localFinish(boolean commit) throws IgniteCheckedException;
+ public boolean localFinish(boolean commit, boolean clearThreadMap) throws IgniteCheckedException;
}
[05/35] ignite git commit: IGNITE-6228 Avoid closing page store by
thread interruption. Fixes #2715
Posted by sb...@apache.org.
IGNITE-6228 Avoid closing page store by thread interruption. Fixes #2715
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e8379aea
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e8379aea
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e8379aea
Branch: refs/heads/ignite-3479
Commit: e8379aeae41fda6fa9fdad187dbbb18f3e92a9cb
Parents: f8ae8f9
Author: Alexei Scherbakov <al...@gmail.com>
Authored: Thu Sep 21 17:40:16 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Thu Sep 21 18:05:45 2017 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 5 +
.../PersistentStoreConfiguration.java | 6 +-
.../GridCacheDatabaseSharedManager.java | 7 +-
.../cache/persistence/file/AsyncFileIO.java | 218 +++++++++++++++++++
.../persistence/file/AsyncFileIOFactory.java | 52 +++++
.../cache/persistence/file/FileIOFactory.java | 25 ++-
.../cache/persistence/file/FilePageStore.java | 5 +-
.../file/FileVersionCheckingFactory.java | 2 +-
.../persistence/file/RandomAccessFileIO.java | 48 ++--
.../file/RandomAccessFileIOFactory.java | 14 +-
.../wal/AbstractWalRecordsIterator.java | 2 +-
.../cache/persistence/wal/FileInput.java | 7 +
.../wal/FileWriteAheadLogManager.java | 8 +-
.../reader/StandaloneWalRecordsIterator.java | 4 +-
.../internal/util/future/GridFutureAdapter.java | 16 ++
.../resources/META-INF/classnames.properties | 2 +
.../file/IgnitePdsThreadInterruptionTest.java | 205 +++++++++++++++++
.../db/wal/IgniteWalFlushFailoverTest.java | 22 +-
.../db/wal/crc/IgniteDataIntegrityTests.java | 39 ++--
.../development/utils/IgniteWalConverter.java | 1 -
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 +
21 files changed, 606 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index ec79026..5f1839b 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -714,6 +714,11 @@ public final class IgniteSystemProperties {
"IGNITE_DATA_STREAMING_EXECUTOR_SERVICE_TASKS_STEALING_THRESHOLD";
/**
+ * If this property is set, then Ignite will use Async File IO factory by default.
+ */
+ public static final String IGNITE_USE_ASYNC_FILE_IO_FACTORY = "IGNITE_USE_ASYNC_FILE_IO_FACTORY";
+
+ /**
* If the property is set {@link org.apache.ignite.internal.pagemem.wal.record.TxRecord} records
* will be logged to WAL.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
index 888bf42..abca5a5 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/PersistentStoreConfiguration.java
@@ -17,6 +17,8 @@
package org.apache.ignite.configuration;
import java.io.Serializable;
+import org.apache.ignite.IgniteSystemProperties;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.util.typedef.internal.S;
@@ -137,7 +139,9 @@ public class PersistentStoreConfiguration implements Serializable {
private boolean alwaysWriteFullPages = DFLT_WAL_ALWAYS_WRITE_FULL_PAGES;
/** Factory to provide I/O interface for files */
- private FileIOFactory fileIOFactory = new RandomAccessFileIOFactory();
+ private FileIOFactory fileIOFactory =
+ IgniteSystemProperties.getBoolean(IgniteSystemProperties.IGNITE_USE_ASYNC_FILE_IO_FACTORY, false) ?
+ new AsyncFileIOFactory() : new RandomAccessFileIOFactory();
/**
* Number of sub-intervals the whole {@link #setRateTimeInterval(long)} will be split into to calculate
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 5a772b5..5f03d8f 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -143,6 +143,7 @@ import org.apache.ignite.thread.IgniteThreadPoolExecutor;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static java.nio.file.StandardOpenOption.READ;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_WAL_REBALANCE_THRESHOLD;
@@ -744,7 +745,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
* @param partFile Partition file.
*/
private int resolvePageSizeFromPartitionFile(Path partFile) throws IOException, IgniteCheckedException {
- try (FileIO fileIO = persistenceCfg.getFileIOFactory().create(partFile.toFile(), "r")) {
+ try (FileIO fileIO = persistenceCfg.getFileIOFactory().create(partFile.toFile())) {
int minimalHdr = FilePageStore.HEADER_SIZE;
if (fileIO.size() < minimalHdr)
@@ -964,7 +965,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
checkpointLock.readLock().unlock();
try {
- checkpointer.wakeupForCheckpoint(0, "too many dirty pages").cpBeginFut.get();
+ checkpointer.wakeupForCheckpoint(0, "too many dirty pages").cpBeginFut.getUninterruptibly();
}
catch (IgniteCheckedException e) {
throw new IgniteException("Failed to wait for checkpoint begin.", e);
@@ -1394,7 +1395,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
private WALPointer readPointer(File cpMarkerFile, ByteBuffer buf) throws IgniteCheckedException {
buf.position(0);
- try (FileChannel ch = FileChannel.open(cpMarkerFile.toPath(), StandardOpenOption.READ)) {
+ try (FileChannel ch = FileChannel.open(cpMarkerFile.toPath(), READ)) {
ch.read(buf);
buf.flip();
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
new file mode 100644
index 0000000..8fad7a5
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIO.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.channels.CompletionHandler;
+import java.nio.file.OpenOption;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.util.GridConcurrentHashSet;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+
+/**
+ * File I/O implementation based on {@link AsynchronousFileChannel}.
+ */
+public class AsyncFileIO implements FileIO {
+ /**
+ * File channel associated with {@code file}
+ */
+ private final AsynchronousFileChannel ch;
+
+ /**
+ * Channel's position.
+ */
+ private volatile long position;
+
+ /** */
+ private final ThreadLocal<ChannelOpFuture> holder;
+
+ /** */
+ private GridConcurrentHashSet<ChannelOpFuture> asyncFuts = new GridConcurrentHashSet<>();
+
+ /**
+ * Creates I/O implementation for specified {@code file}
+ * @param file Random access file
+ * @param modes Open modes.
+ */
+ public AsyncFileIO(File file, ThreadLocal<ChannelOpFuture> holder, OpenOption... modes) throws IOException {
+ this.ch = AsynchronousFileChannel.open(file.toPath(), modes);
+
+ this.holder = holder;
+ }
+
+ /** {@inheritDoc} */
+ @Override public long position() throws IOException {
+ return position;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void position(long newPosition) throws IOException {
+ this.position = newPosition;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(ByteBuffer destinationBuffer) throws IOException {
+ ChannelOpFuture fut = holder.get();
+ fut.reset();
+
+ ch.read(destinationBuffer, position, this, fut);
+
+ try {
+ return fut.getUninterruptibly();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(ByteBuffer destinationBuffer, long position) throws IOException {
+ ChannelOpFuture fut = holder.get();
+ fut.reset();
+
+ ch.read(destinationBuffer, position, null, fut);
+
+ try {
+ return fut.getUninterruptibly();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ finally {
+ asyncFuts.remove(fut);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(byte[] buffer, int offset, int length) throws IOException {
+ ChannelOpFuture fut = holder.get();
+ fut.reset();
+
+ ch.read(ByteBuffer.wrap(buffer, offset, length), position, this, fut);
+
+ try {
+ return fut.getUninterruptibly();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int write(ByteBuffer sourceBuffer) throws IOException {
+ ChannelOpFuture fut = holder.get();
+ fut.reset();
+
+ ch.write(sourceBuffer, position, this, fut);
+
+ try {
+ return fut.getUninterruptibly();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public int write(ByteBuffer sourceBuffer, long position) throws IOException {
+ ChannelOpFuture fut = holder.get();
+ fut.reset();
+
+ asyncFuts.add(fut);
+
+ ch.write(sourceBuffer, position, null, fut);
+
+ try {
+ return fut.getUninterruptibly();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ finally {
+ asyncFuts.remove(fut);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void write(byte[] buffer, int offset, int length) throws IOException {
+ ChannelOpFuture fut = holder.get();
+ fut.reset();
+
+ ch.write(ByteBuffer.wrap(buffer, offset, length), position, this, fut);
+
+ try {
+ fut.getUninterruptibly();
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void force() throws IOException {
+ ch.force(false);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long size() throws IOException {
+ return ch.size();
+ }
+
+ /** {@inheritDoc} */
+ @Override public void clear() throws IOException {
+ ch.truncate(0);
+
+ this.position = 0;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IOException {
+ for (ChannelOpFuture asyncFut : asyncFuts) {
+ try {
+ asyncFut.getUninterruptibly(); // Ignore interrupts while waiting for channel close.
+ }
+ catch (IgniteCheckedException e) {
+ throw new IOException(e);
+ }
+ }
+
+ ch.close();
+ }
+
+ /** */
+ static class ChannelOpFuture extends GridFutureAdapter<Integer> implements CompletionHandler<Integer, AsyncFileIO> {
+ /** {@inheritDoc} */
+ @Override public void completed(Integer result, AsyncFileIO attachment) {
+ if (attachment != null) {
+ if (result != -1)
+ attachment.position += result;
+ }
+
+ // Release waiter and allow next operation to begin.
+ super.onDone(result, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void failed(Throwable exc, AsyncFileIO attachment) {
+ super.onDone(exc);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java
new file mode 100644
index 0000000..0fb3052
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/AsyncFileIOFactory.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.channels.AsynchronousFileChannel;
+import java.nio.file.OpenOption;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+/**
+ * File I/O factory which uses {@link AsynchronousFileChannel} based implementation of FileIO.
+ */
+public class AsyncFileIOFactory implements FileIOFactory {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return create(file, CREATE, READ, WRITE);
+ }
+
+ /** */
+ private ThreadLocal<AsyncFileIO.ChannelOpFuture> holder = new ThreadLocal<AsyncFileIO.ChannelOpFuture>() {
+ @Override protected AsyncFileIO.ChannelOpFuture initialValue() {
+ return new AsyncFileIO.ChannelOpFuture();
+ }
+ };
+
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ return new AsyncFileIO(file, holder, modes);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
index 0ffc653..c3a75f5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileIOFactory.java
@@ -20,26 +20,29 @@ package org.apache.ignite.internal.processors.cache.persistence.file;
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
+import java.nio.file.OpenOption;
+/**
+ * {@link FileIO} factory definition.
+ */
public interface FileIOFactory extends Serializable {
-
/**
- * Creates I/O interface for file with default I/O mode
+ * Creates I/O interface for file with default I/O mode.
*
- * @param file File
- * @return File I/O interface
- * @throws IOException If I/O interface creation was failed
+ * @param file File.
+ * @return File I/O interface.
+ * @throws IOException If I/O interface creation was failed.
*/
- FileIO create(File file) throws IOException;
+ public FileIO create(File file) throws IOException;
/**
- * Creates I/O interface for file with specified mode
+ * Creates I/O interface for file with specified mode.
*
* @param file File
- * @param mode I/O mode in
- * @return File I/O interface
- * @throws IOException If I/O interface creation was failed
+ * @param modes Open modes.
+ * @return File I/O interface.
+ * @throws IOException If I/O interface creation was failed.
*/
- FileIO create(File file, String mode) throws IOException;
+ public FileIO create(File file, OpenOption... modes) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 6052a7c..98764a2 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -35,6 +35,9 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDat
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.PureJavaCrc32;
import org.apache.ignite.internal.util.typedef.internal.U;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_PDS_SKIP_CRC;
/**
@@ -400,7 +403,7 @@ public class FilePageStore implements PageStore {
IgniteCheckedException err = null;
try {
- this.fileIO = fileIO = ioFactory.create(cfgFile, "rw");
+ this.fileIO = fileIO = ioFactory.create(cfgFile, CREATE, READ, WRITE);
if (cfgFile.length() == 0)
allocated.set(initFile());
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
index 53bd802..40870dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FileVersionCheckingFactory.java
@@ -55,7 +55,7 @@ public class FileVersionCheckingFactory implements FilePageStoreFactory {
if (!file.exists())
return createPageStore(type, file, latestVersion());
- try (FileIO fileIO = fileIOFactory.create(file, "r")) {
+ try (FileIO fileIO = fileIOFactory.create(file)) {
int minHdr = FilePageStore.HEADER_SIZE;
if (fileIO.size() < minHdr)
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
index 73a560a..55849fe 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIO.java
@@ -17,94 +17,88 @@
package org.apache.ignite.internal.processors.cache.persistence.file;
+import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.nio.file.OpenOption;
/**
- * File I/O implementation based on {@code java.io.RandomAccessFile}.
+ * File I/O implementation based on {@link FileChannel}.
*/
public class RandomAccessFileIO implements FileIO {
-
- /**
- * Random access file associated with this I/O
- */
- private final RandomAccessFile file;
-
/**
- * File channel associated with {@code file}
+ * File channel.
*/
- private final FileChannel channel;
+ private final FileChannel ch;
/**
* Creates I/O implementation for specified {@code file}
*
- * @param file Random access file
+ * @param file File.
+ * @param modes Open modes.
*/
- public RandomAccessFileIO(RandomAccessFile file) {
- this.file = file;
- this.channel = file.getChannel();
+ public RandomAccessFileIO(File file, OpenOption... modes) throws IOException {
+ ch = FileChannel.open(file.toPath(), modes);
}
/** {@inheritDoc} */
@Override public long position() throws IOException {
- return channel.position();
+ return ch.position();
}
/** {@inheritDoc} */
@Override public void position(long newPosition) throws IOException {
- channel.position(newPosition);
+ ch.position(newPosition);
}
/** {@inheritDoc} */
@Override public int read(ByteBuffer destinationBuffer) throws IOException {
- return channel.read(destinationBuffer);
+ return ch.read(destinationBuffer);
}
/** {@inheritDoc} */
@Override public int read(ByteBuffer destinationBuffer, long position) throws IOException {
- return channel.read(destinationBuffer, position);
+ return ch.read(destinationBuffer, position);
}
/** {@inheritDoc} */
@Override public int read(byte[] buffer, int offset, int length) throws IOException {
- return file.read(buffer, offset, length);
+ return ch.read(ByteBuffer.wrap(buffer, offset, length));
}
/** {@inheritDoc} */
@Override public int write(ByteBuffer sourceBuffer) throws IOException {
- return channel.write(sourceBuffer);
+ return ch.write(sourceBuffer);
}
/** {@inheritDoc} */
@Override public int write(ByteBuffer sourceBuffer, long position) throws IOException {
- return channel.write(sourceBuffer, position);
+ return ch.write(sourceBuffer, position);
}
/** {@inheritDoc} */
@Override public void write(byte[] buffer, int offset, int length) throws IOException {
- file.write(buffer, offset, length);
+ ch.write(ByteBuffer.wrap(buffer, offset, length));
}
/** {@inheritDoc} */
@Override public void force() throws IOException {
- channel.force(false);
+ ch.force(false);
}
/** {@inheritDoc} */
@Override public long size() throws IOException {
- return channel.size();
+ return ch.size();
}
/** {@inheritDoc} */
@Override public void clear() throws IOException {
- channel.position(0);
- file.setLength(0);
+ ch.truncate(0);
}
/** {@inheritDoc} */
@Override public void close() throws IOException {
- file.close();
+ ch.close();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
index 6b731f2..856ba1c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/RandomAccessFileIOFactory.java
@@ -19,7 +19,11 @@ package org.apache.ignite.internal.processors.cache.persistence.file;
import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
+import java.nio.file.OpenOption;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
/**
* File I/O factory which provides RandomAccessFileIO implementation of FileIO.
@@ -30,13 +34,11 @@ public class RandomAccessFileIOFactory implements FileIOFactory {
/** {@inheritDoc} */
@Override public FileIO create(File file) throws IOException {
- return create(file, "rw");
+ return create(file, CREATE, READ, WRITE);
}
/** {@inheritDoc} */
- @Override public FileIO create(File file, String mode) throws IOException {
- RandomAccessFile rf = new RandomAccessFile(file, mode);
-
- return new RandomAccessFileIO(rf);
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ return new RandomAccessFileIO(file, modes);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
index 2749d5c..d5a2555 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/AbstractWalRecordsIterator.java
@@ -258,7 +258,7 @@ public abstract class AbstractWalRecordsIterator
@Nullable final FileWALPointer start)
throws IgniteCheckedException, FileNotFoundException {
try {
- FileIO fileIO = ioFactory.create(desc.file, "r");
+ FileIO fileIO = ioFactory.create(desc.file);
try {
FileInput in = new FileInput(fileIO, buf);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
index 74edbfa..3b20fce 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileInput.java
@@ -63,6 +63,13 @@ public final class FileInput implements ByteBufferBackedDataInput {
}
/**
+ * File I/O.
+ */
+ public FileIO io() {
+ return io;
+ }
+
+ /**
* Clear buffer.
*/
private void clearBuffer() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 18584a8..87069d9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -79,6 +79,10 @@ import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
/**
* File WAL manager.
*/
@@ -940,7 +944,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (log.isDebugEnabled())
log.debug("Formatting file [exists=" + file.exists() + ", file=" + file.getAbsolutePath() + ']');
- try (FileIO fileIO = ioFactory.create(file, "rw")) {
+ try (FileIO fileIO = ioFactory.create(file, CREATE, READ, WRITE)) {
int left = psCfg.getWalSegmentSize();
if (mode == WALMode.DEFAULT) {
@@ -1365,7 +1369,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
Files.move(dstTmpFile.toPath(), dstFile.toPath());
if (mode == WALMode.DEFAULT) {
- try (FileIO f0 = ioFactory.create(dstFile, "rw")) {
+ try (FileIO f0 = ioFactory.create(dstFile, CREATE, READ, WRITE)) {
f0.force();
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
index 900aab5..c92d572 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneWalRecordsIterator.java
@@ -181,7 +181,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
/**
* This methods checks all provided files to be correct WAL segment.
- * Header record and its position is checked. WAL position is used to deremine real index.
+ * Header record and its position is checked. WAL position is used to determine real index.
* File index from file name is ignored.
*
* @param allFiles files to scan
@@ -202,7 +202,7 @@ class StandaloneWalRecordsIterator extends AbstractWalRecordsIterator {
FileWALPointer ptr;
try (
- FileIO fileIO = ioFactory.create(file, "r");
+ FileIO fileIO = ioFactory.create(file);
ByteBufferExpander buf = new ByteBufferExpander(HEADER_RECORD_SIZE, ByteOrder.nativeOrder())
) {
final DataInput in = new FileInput(fileIO, buf);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
index 323babd..f8c0b14 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/future/GridFutureAdapter.java
@@ -22,6 +22,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.concurrent.locks.LockSupport;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.internal.IgniteFutureCancelledCheckedException;
import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
@@ -499,6 +500,21 @@ public class GridFutureAdapter<R> implements IgniteInternalFuture<R> {
}
/**
+ * Resets future for subsequent reuse.
+ */
+ public void reset() {
+ final Object oldState = state;
+
+ if (oldState == INIT)
+ return;
+
+ if (!isDone(oldState))
+ throw new IgniteException("Illegal state");
+
+ compareAndSetState(oldState, INIT);
+ }
+
+ /**
* Callback to notify that future is cancelled.
*
* @return {@code True} if cancel flag was set by this call.
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/main/resources/META-INF/classnames.properties
----------------------------------------------------------------------
diff --git a/modules/core/src/main/resources/META-INF/classnames.properties b/modules/core/src/main/resources/META-INF/classnames.properties
index 1931a93..d91eef4 100644
--- a/modules/core/src/main/resources/META-INF/classnames.properties
+++ b/modules/core/src/main/resources/META-INF/classnames.properties
@@ -2091,3 +2091,5 @@ org.apache.ignite.transactions.TransactionRollbackException
org.apache.ignite.transactions.TransactionState
org.apache.ignite.transactions.TransactionTimeoutException
org.apache.ignite.util.AttributeNodeFilter
+org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIO
+org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java
new file mode 100644
index 0000000..aab569a
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsThreadInterruptionTest.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.file;
+
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.MemoryConfiguration;
+import org.apache.ignite.configuration.MemoryPolicyConfiguration;
+import org.apache.ignite.configuration.PersistentStoreConfiguration;
+import org.apache.ignite.configuration.WALMode;
+import org.apache.ignite.internal.processors.cache.persistence.file.AsyncFileIOFactory;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.jsr166.ThreadLocalRandom8;
+
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Test what interruptions of writing threads do not affect PDS.
+ */
+public class IgnitePdsThreadInterruptionTest extends GridCommonAbstractTest {
+ /** */
+ private static final int PAGE_SIZE = 1 << 12; // 4096
+
+ /** */
+ public static final int THREADS_CNT = 1;
+
+ /**
+ * Cache name.
+ */
+ private final String cacheName = "cache";
+
+ /** */
+ private volatile boolean stop = false;
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ final IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.setPersistentStoreConfiguration(storeConfiguration());
+
+ cfg.setMemoryConfiguration(memoryConfiguration());
+
+ cfg.setCacheConfiguration(new CacheConfiguration<>(cacheName));
+
+ return cfg;
+ }
+
+ /**
+ * @return Store config.
+ */
+ private PersistentStoreConfiguration storeConfiguration() {
+ PersistentStoreConfiguration cfg = new PersistentStoreConfiguration();
+
+ cfg.setWalMode(WALMode.LOG_ONLY);
+
+ cfg.setWalFsyncDelayNanos(0);
+
+ cfg.setFileIOFactory(new AsyncFileIOFactory());
+
+ return cfg;
+ }
+
+ /**
+ * @return Memory config.
+ */
+ private MemoryConfiguration memoryConfiguration() {
+ final MemoryConfiguration memCfg = new MemoryConfiguration();
+
+ MemoryPolicyConfiguration memPlcCfg = new MemoryPolicyConfiguration();
+ // memPlcCfg.setPageEvictionMode(RANDOM_LRU); TODO Fix NPE on start.
+ memPlcCfg.setName("dfltMemPlc");
+
+ memCfg.setPageSize(PAGE_SIZE);
+ memCfg.setConcurrencyLevel(1);
+ memCfg.setMemoryPolicies(memPlcCfg);
+ memCfg.setDefaultMemoryPolicyName("dfltMemPlc");
+
+ return memCfg;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ stopAllGrids();
+
+ deleteWorkFiles();
+ }
+
+ /**
+ * Tests interruptions on WAL write.
+ *
+ * @throws Exception
+ */
+ public void testInterruptsOnWALWrite() throws Exception {
+ final Ignite ignite = startGrid();
+
+ ignite.active(true);
+
+ final int valLen = 8192;
+
+ final byte[] payload = new byte[valLen];
+
+ final int maxKey = 100_000;
+
+ Thread[] workers = new Thread[THREADS_CNT];
+
+ final AtomicReference<Throwable> fail = new AtomicReference<>();
+
+ Runnable clo = new Runnable() {
+ @Override public void run() {
+ IgniteCache<Object, Object> cache = ignite.cache(cacheName);
+
+ while (!stop)
+ cache.put(ThreadLocalRandom8.current().nextInt(maxKey), payload);
+ }
+ };
+
+ for (int i = 0; i < workers.length; i++) {
+ workers[i] = new Thread(clo);
+ workers[i].setName("writer-" + i);
+ workers[i].setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
+ @Override public void uncaughtException(Thread t, Throwable e) {
+ fail.compareAndSet(null, e);
+ }
+ });
+ }
+
+ for (Thread worker : workers)
+ worker.start();
+
+ Thread.sleep(3_000);
+
+ // Interrupts should not affect writes.
+ for (Thread worker : workers)
+ worker.interrupt();
+
+ Thread.sleep(3_000);
+
+ stop = true;
+
+ for (Thread worker : workers)
+ worker.join();
+
+ Throwable t = fail.get();
+
+ assert t == null : t;
+
+ IgniteCache<Object, Object> cache = ignite.cache(cacheName);
+
+ int verifiedKeys = 0;
+
+ // Post check.
+ for (int i = 0; i < maxKey; i++) {
+ byte[] val = (byte[]) cache.get(i);
+
+ if (val != null) {
+ assertEquals("Illegal length", valLen, val.length);
+
+ verifiedKeys++;
+ }
+ }
+
+ log.info("Verified keys: " + verifiedKeys);
+ }
+
+ /**
+ * @throws IgniteCheckedException If fail.
+ */
+ private void deleteWorkFiles() throws IgniteCheckedException {
+ deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), "db", false));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
index cad10ae..048e8bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/IgniteWalFlushFailoverTest.java
@@ -42,12 +42,16 @@ import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
+import java.nio.file.OpenOption;
+
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
/**
*
*/
public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
-
/** */
private static final String TEST_CACHE = "testCache";
@@ -168,22 +172,22 @@ public class IgniteWalFlushFailoverTest extends GridCommonAbstractTest {
private static class FailingFileIOFactory implements FileIOFactory {
private static final long serialVersionUID = 0L;
+ /** */
private final FileIOFactory delegateFactory = new RandomAccessFileIOFactory();
- @Override
- public FileIO create(File file) throws IOException {
- return create(file, "rw");
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file) throws IOException {
+ return create(file, CREATE, READ, WRITE);
}
- @Override
- public FileIO create(File file, String mode) throws IOException {
- FileIO delegate = delegateFactory.create(file, mode);
+ /** {@inheritDoc} */
+ @Override public FileIO create(File file, OpenOption... modes) throws IOException {
+ FileIO delegate = delegateFactory.create(file, modes);
return new FileIODecorator(delegate) {
int writeAttempts = 2;
- @Override
- public int write(ByteBuffer sourceBuffer) throws IOException {
+ @Override public int write(ByteBuffer sourceBuffer) throws IOException {
if (--writeAttempts == 0)
throw new RuntimeException("Test exception. Unable to write to file.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
index e4874d9..3d52507 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/crc/IgniteDataIntegrityTests.java
@@ -21,11 +21,11 @@ import junit.framework.TestCase;
import java.io.EOFException;
import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.concurrent.ThreadLocalRandom;
-import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIO;
+import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactory;
+import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.wal.ByteBufferExpander;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileInput;
import org.apache.ignite.internal.processors.cache.persistence.wal.crc.IgniteDataIntegrityViolationException;
@@ -38,9 +38,6 @@ public class IgniteDataIntegrityTests extends TestCase {
/** File input. */
private FileInput fileInput;
- /** Random access file. */
- private RandomAccessFile randomAccessFile;
-
/** Buffer expander. */
private ByteBufferExpander expBuf;
@@ -51,13 +48,13 @@ public class IgniteDataIntegrityTests extends TestCase {
File file = File.createTempFile("integrity", "dat");
file.deleteOnExit();
- randomAccessFile = new RandomAccessFile(file, "rw");
-
expBuf = new ByteBufferExpander(1024, ByteOrder.BIG_ENDIAN);
+ FileIOFactory factory = new RandomAccessFileIOFactory();
+
fileInput = new FileInput(
- new RandomAccessFileIO(randomAccessFile),
- expBuf
+ factory.create(file),
+ expBuf
);
ByteBuffer buf = ByteBuffer.allocate(1024);
@@ -71,13 +68,15 @@ public class IgniteDataIntegrityTests extends TestCase {
buf.putInt(PureJavaCrc32.calcCrc32(buf, 12));
}
- randomAccessFile.write(buf.array());
- randomAccessFile.getFD().sync();
+ buf.rewind();
+
+ fileInput.io().write(buf);
+ fileInput.io().force();
}
/** {@inheritDoc} */
@Override protected void tearDown() throws Exception {
- randomAccessFile.close();
+ fileInput.io().close();
expBuf.close();
}
@@ -177,22 +176,24 @@ public class IgniteDataIntegrityTests extends TestCase {
*/
private void toggleOneRandomBit(int rangeFrom, int rangeTo) throws IOException {
int pos = ThreadLocalRandom.current().nextInt(rangeFrom, rangeTo);
- randomAccessFile.seek(pos);
+ fileInput.io().position(pos);
+
+ byte[] buf = new byte[1];
- byte b = randomAccessFile.readByte();
+ fileInput.io().read(buf, 0, 1);
- b ^= (1 << 3);
+ buf[0] ^= (1 << 3);
- randomAccessFile.seek(pos);
- randomAccessFile.writeByte(b);
- randomAccessFile.getFD().sync();
+ fileInput.io().position(pos);
+ fileInput.io().write(buf, 0, 1);
+ fileInput.io().force();
}
/**
*
*/
private void checkIntegrity() throws Exception {
- randomAccessFile.seek(0);
+ fileInput.io().position(0);
for (int i = 0; i < 1024 / 16; i++) {
try(FileInput.Crc32CheckingFileInput in = fileInput.startRead(false)) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
----------------------------------------------------------------------
diff --git a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
index b7c3cb7..f3268d9 100644
--- a/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
+++ b/modules/dev-utils/src/main/java/org/apache/ignite/development/utils/IgniteWalConverter.java
@@ -21,7 +21,6 @@ import java.io.File;
import org.apache.ignite.internal.pagemem.wal.WALIterator;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.pagemem.wal.record.WALRecord;
-import org.apache.ignite.internal.processors.cache.persistence.file.RandomAccessFileIOFactory;
import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO;
import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager;
import org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e8379aea/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index ae8ea18..cfbe2e0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -26,6 +26,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsMulti
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsPageEvictionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsCacheIntegrationTest;
import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsNoActualWalHistoryTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.file.IgnitePdsThreadInterruptionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest;
@@ -53,6 +54,7 @@ public class IgnitePdsWithIndexingCoreTestSuite extends TestSuite {
suite.addTestSuite(IgnitePdsBinaryMetadataOnClusterRestartTest.class);
suite.addTestSuite(IgnitePdsMarshallerMappingRestoreOnNodeStartTest.class);
+ suite.addTestSuite(IgnitePdsThreadInterruptionTest.class);
return suite;
}
[30/35] ignite git commit: ignite-3478
Posted by sb...@apache.org.
ignite-3478
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e4494600
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e4494600
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e4494600
Branch: refs/heads/ignite-3479
Commit: e449460082f292e54060dcc7786ffa03986b3890
Parents: b72f362
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 13:26:56 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 13:26:56 2017 +0300
----------------------------------------------------------------------
.../cache/mvcc/CacheMvccTransactionsTest.java | 121 ++++++++++++++++++-
1 file changed, 119 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e4494600/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
index c50d63c..e5eb0ee 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
@@ -702,8 +702,6 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
assertEquals(5, (Object)vals.get(key2));
}
-
-
/**
* @throws Exception If failed.
*/
@@ -812,6 +810,125 @@ public class CacheMvccTransactionsTest extends GridCommonAbstractTest {
/**
* @throws Exception If failed.
*/
+ public void testCleanupWaitsForGet3() throws Exception {
+ /*
+ Simulate case when coordinator assigned query version has active transaction,
+ query is delayed, after this active transaction finish and the same key is
+ updated several more times before query starts.
+ */
+ testSpi = true;
+
+ client = false;
+
+ startGrids(1);
+
+ client = true;
+
+ final Ignite client = startGrid(1);
+
+ awaitPartitionMapExchange();
+
+ final IgniteCache<Object, Object> cache = client.createCache(cacheConfiguration(PARTITIONED, FULL_SYNC, 0, 16));
+
+ final Integer key1 = 1;
+ final Integer key2 = 2;
+
+ for (int i = 0; i < 3; i++) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(key1, i);
+ cache.put(key2, i);
+
+ tx.commit();
+ }
+ }
+
+ TestRecordingCommunicationSpi crdSpi = TestRecordingCommunicationSpi.spi(grid(0));
+
+ TestRecordingCommunicationSpi clientSpi = TestRecordingCommunicationSpi.spi(client);
+
+ clientSpi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ /** */
+ private boolean blocked;
+
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ if (!blocked && (msg instanceof CoordinatorTxAckRequest)) {
+ blocked = true;
+
+ return true;
+ }
+ return false;
+ }
+ });
+
+ final IgniteInternalFuture<?> putFut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(key2, 3);
+
+ tx.commit();
+ }
+
+ return null;
+ }
+ }, "put");
+
+ clientSpi.waitForBlocked();
+
+ for (int i = 0; i < 3; i++) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(key1, i + 3);
+
+ tx.commit();
+ }
+ }
+
+ // Delay version for getAll.
+ crdSpi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
+ /** */
+ private boolean blocked;
+
+ @Override public boolean apply(ClusterNode node, Message msg) {
+ if (!blocked && (msg instanceof MvccCoordinatorVersionResponse)) {
+ blocked = true;
+
+ return true;
+ }
+ return false;
+ }
+ });
+
+ final IgniteInternalFuture<?> getFut = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ Map<Object, Object> res = cache.getAll(F.asSet(key1, key2));
+
+ assertEquals(2, res.size());
+
+ return null;
+ }
+ }, "get");
+
+ crdSpi.waitForBlocked();
+
+ clientSpi.stopBlock(true);
+
+ putFut.get();
+
+ for (int i = 0; i < 3; i++) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(key2, i + 4);
+
+ tx.commit();
+ }
+ }
+
+ crdSpi.stopBlock(true);
+
+ getFut.get();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
public void testPutAllGetAll_SingleNode() throws Exception {
putAllGetAll(1, 0, 0, 64);
}
[29/35] ignite git commit: IGNITE-6480 fix serverTopologyNodes method
- Fixes #2726.
Posted by sb...@apache.org.
IGNITE-6480 fix serverTopologyNodes method - Fixes #2726.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/05eba9a2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/05eba9a2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/05eba9a2
Branch: refs/heads/ignite-3479
Commit: 05eba9a2e19614843157a04fe33c281be65c7d52
Parents: e3d448e
Author: Dmitriy Govorukhin <dm...@gmail.com>
Authored: Fri Sep 22 13:14:57 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 22 13:14:57 2017 +0300
----------------------------------------------------------------------
.../internal/managers/discovery/GridDiscoveryManager.java | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/05eba9a2/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index 33350de..527399d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -181,7 +181,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
private static final int DISCOVERY_HISTORY_SIZE = getInteger(IGNITE_DISCOVERY_HISTORY_SIZE, 500);
/** Predicate filtering out daemon nodes. */
- private static final IgnitePredicate<ClusterNode> FILTER_DAEMON = new P1<ClusterNode>() {
+ private static final IgnitePredicate<ClusterNode> FILTER_NOT_DAEMON = new P1<ClusterNode>() {
@Override public boolean apply(ClusterNode n) {
return !n.isDaemon();
}
@@ -736,7 +736,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
discoEvt.eventNode(node);
discoEvt.type(EVT_NODE_JOINED);
- discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(topSnapshot, FILTER_DAEMON)));
+ discoEvt.topologySnapshot(topVer, new ArrayList<>(F.view(topSnapshot, FILTER_NOT_DAEMON)));
discoWrk.discoCache = discoCache;
@@ -2002,7 +2002,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
* @return Server topology nodes or {@code null} if there are no nodes for passed in version.
*/
@Nullable public Collection<ClusterNode> serverTopologyNodes(long topVer) {
- return F.view(topology(topVer), F.not(FILTER_CLI));
+ return F.view(topology(topVer), F.not(FILTER_CLI), FILTER_NOT_DAEMON);
}
/** @return All daemon nodes in topology. */
@@ -2456,7 +2456,7 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> {
evt.node(ctx.discovery().localNode());
evt.eventNode(node);
evt.type(type);
- evt.topologySnapshot(topVer, U.<ClusterNode, ClusterNode>arrayList(topSnapshot, FILTER_DAEMON));
+ evt.topologySnapshot(topVer, U.<ClusterNode, ClusterNode>arrayList(topSnapshot, FILTER_NOT_DAEMON));
if (type == EVT_NODE_METRICS_UPDATED)
evt.message("Metrics were updated: " + node);
[19/35] ignite git commit: IGNITE-6213 Removed locDepOwner flag and
marked as deprecated - Fixes #2723.
Posted by sb...@apache.org.
IGNITE-6213 Removed locDepOwner flag and marked as deprecated - Fixes #2723.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1649c532
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1649c532
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1649c532
Branch: refs/heads/ignite-3479
Commit: 1649c532f828cef65c494ad7b008d9f1a8f3d605
Parents: 5af30cf
Author: Dmitriy Govorukhin <dm...@gmail.com>
Authored: Fri Sep 22 12:11:15 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 22 12:11:15 2017 +0300
----------------------------------------------------------------------
.../internal/GridEventConsumeHandler.java | 9 +++-
.../managers/deployment/GridDeploymentInfo.java | 1 +
.../deployment/GridDeploymentInfoBean.java | 21 +++-----
.../cache/GridCacheDeploymentManager.java | 54 ++++++--------------
.../processors/cache/GridCacheIoManager.java | 9 +++-
.../continuous/CacheContinuousQueryHandler.java | 9 +++-
6 files changed, 45 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1649c532/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
index 54e3d7f..ac568f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/GridEventConsumeHandler.java
@@ -321,8 +321,13 @@ class GridEventConsumeHandler implements GridContinuousHandler {
GridDeploymentInfo depInfo = wrapper.depInfo;
if (depInfo != null) {
- depMgr.p2pContext(nodeId, depInfo.classLoaderId(), depInfo.userVersion(), depInfo.deployMode(),
- depInfo.participants(), depInfo.localDeploymentOwner());
+ depMgr.p2pContext(
+ nodeId,
+ depInfo.classLoaderId(),
+ depInfo.userVersion(),
+ depInfo.deployMode(),
+ depInfo.participants()
+ );
}
ldr = depMgr.globalLoader();
http://git-wip-us.apache.org/repos/asf/ignite/blob/1649c532/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfo.java
index b14fcce..3396d82 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfo.java
@@ -49,6 +49,7 @@ public interface GridDeploymentInfo {
/**
* @return Local deployment ownership flag.
*/
+ @Deprecated
public boolean localDeploymentOwner();
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/1649c532/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
index 68f45da..7f58ce3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/deployment/GridDeploymentInfoBean.java
@@ -52,6 +52,7 @@ public class GridDeploymentInfoBean implements Message, GridDeploymentInfo, Exte
private String userVer;
/** */
+ @Deprecated // Left for backward compatibility only.
private boolean locDepOwner;
/** Node class loader participant map. */
@@ -71,15 +72,17 @@ public class GridDeploymentInfoBean implements Message, GridDeploymentInfo, Exte
* @param userVer User version.
* @param depMode Deployment mode.
* @param participants Participants.
- * @param locDepOwner Local deployment owner flag.
*/
- public GridDeploymentInfoBean(IgniteUuid clsLdrId, String userVer, DeploymentMode depMode,
- Map<UUID, IgniteUuid> participants, boolean locDepOwner) {
+ public GridDeploymentInfoBean(
+ IgniteUuid clsLdrId,
+ String userVer,
+ DeploymentMode depMode,
+ Map<UUID, IgniteUuid> participants
+ ) {
this.clsLdrId = clsLdrId;
this.depMode = depMode;
this.userVer = userVer;
this.participants = participants;
- this.locDepOwner = locDepOwner;
}
/**
@@ -89,7 +92,6 @@ public class GridDeploymentInfoBean implements Message, GridDeploymentInfo, Exte
clsLdrId = dep.classLoaderId();
depMode = dep.deployMode();
userVer = dep.userVersion();
- locDepOwner = dep.localDeploymentOwner();
participants = dep.participants();
}
@@ -123,15 +125,6 @@ public class GridDeploymentInfoBean implements Message, GridDeploymentInfo, Exte
return participants;
}
- /**
- * Sets local deployment ownership flag.
- *
- * @param locDepOwner Local deployment ownership flag.
- */
- public void localDeploymentOwner(boolean locDepOwner) {
- this.locDepOwner = locDepOwner;
- }
-
/** {@inheritDoc} */
@Override public void onAckReceived() {
// No-op.
http://git-wip-us.apache.org/repos/asf/ignite/blob/1649c532/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
index a1eb706..c40ed98 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheDeploymentManager.java
@@ -82,9 +82,6 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
/** Local deployment. */
private final AtomicReference<GridDeployment> locDep = new AtomicReference<>();
- /** Local deployment ownership flag. */
- private volatile boolean locDepOwner;
-
/** */
private final ThreadLocal<Boolean> ignoreOwnership = new ThreadLocal<Boolean>() {
@Override protected Boolean initialValue() {
@@ -150,7 +147,7 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
/**
* Gets distributed class loader. Note that
- * {@link #p2pContext(UUID, IgniteUuid, String, DeploymentMode, Map, boolean)} must be
+ * {@link #p2pContext(UUID, IgniteUuid, String, DeploymentMode, Map)} must be
* called from the same thread prior to using this class loader, or the
* loading may happen for the wrong node or context.
*
@@ -164,16 +161,7 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
* Callback on method enter.
*/
public void onEnter() {
- if (depEnabled && !locDepOwner && !ignoreOwnership.get()
- && !cctx.kernalContext().job().internal()) {
- ClassLoader ldr = Thread.currentThread().getContextClassLoader();
-
- // We mark node as deployment owner if accessing cache not from p2p deployed job
- // and not from internal job.
- if (!U.p2pLoader(ldr))
- // If not deployment class loader, classes can be loaded from this node.
- locDepOwner = true;
- }
+ // No-op.
}
/**
@@ -369,10 +357,14 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
* @param userVer User version.
* @param mode Deployment mode.
* @param participants Node participants.
- * @param locDepOwner {@code True} if local deployment owner.
*/
- public void p2pContext(UUID sndId, IgniteUuid ldrId, String userVer, DeploymentMode mode,
- Map<UUID, IgniteUuid> participants, boolean locDepOwner) {
+ public void p2pContext(
+ UUID sndId,
+ IgniteUuid ldrId,
+ String userVer,
+ DeploymentMode mode,
+ Map<UUID, IgniteUuid> participants
+ ) {
assert depEnabled;
if (mode == PRIVATE || mode == ISOLATED) {
@@ -414,7 +406,7 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
if (log.isDebugEnabled())
log.debug("Setting p2p context [sndId=" + sndId + ", ldrId=" + ldrId + ", userVer=" + userVer +
", seqNum=" + ldrId.localId() + ", mode=" + mode + ", participants=" + participants +
- ", locDepOwner=" + locDepOwner + ']');
+ ", locDepOwner=false]");
CachedDeploymentInfo<K, V> depInfo;
@@ -443,18 +435,10 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
break;
}
- Map<UUID, IgniteUuid> added = null;
-
- if (locDepOwner)
- added = addGlobalParticipants(sndId, ldrId, participants, locDepOwner);
-
if (cctx.discovery().node(sndId) == null) {
// Sender has left.
deps.remove(ldrId, depInfo);
- if (added != null)
- added.remove(sndId);
-
allParticipants.remove(sndId);
}
@@ -464,16 +448,11 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
if (depInfo.removeParticipant(id))
deps.remove(ldrId, depInfo);
- if (added != null)
- added.remove(id);
allParticipants.remove(id);
}
}
}
-
- if (added != null && !added.isEmpty())
- cctx.gridDeploy().addCacheParticipants(allParticipants, added);
}
/**
@@ -690,8 +669,6 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
if (locDep0 != null) {
// Will copy sequence number to bean.
dep = new GridDeploymentInfoBean(locDep0);
-
- dep.localDeploymentOwner(locDepOwner);
}
}
@@ -709,9 +686,6 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
@Nullable public GridDeploymentInfoBean globalDeploymentInfo() {
assert depEnabled;
- if (locDepOwner)
- return null;
-
// Do not return info if mode is CONTINUOUS.
// In this case deployment info will be set by GridCacheMessage.prepareObject().
if (cctx.gridConfig().getDeploymentMode() == CONTINUOUS)
@@ -729,8 +703,12 @@ public class GridCacheDeploymentManager<K, V> extends GridCacheSharedManagerAdap
for (UUID id : participants.keySet()) {
if (cctx.discovery().node(id) != null) {
// At least 1 participant is still in the grid.
- return new GridDeploymentInfoBean(d.loaderId(), d.userVersion(), d.mode(),
- participants, locDepOwner);
+ return new GridDeploymentInfoBean(
+ d.loaderId(),
+ d.userVersion(),
+ d.mode(),
+ participants
+ );
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1649c532/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
index 6529795..e6e40e2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java
@@ -1510,8 +1510,13 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter {
assert depEnabled : "Received deployment info while peer class loading is disabled [nodeId=" + nodeId +
", msg=" + cacheMsg + ']';
- cctx.deploy().p2pContext(nodeId, bean.classLoaderId(), bean.userVersion(),
- bean.deployMode(), bean.participants(), bean.localDeploymentOwner());
+ cctx.deploy().p2pContext(
+ nodeId,
+ bean.classLoaderId(),
+ bean.userVersion(),
+ bean.deployMode(),
+ bean.participants()
+ );
if (log.isDebugEnabled())
log.debug("Set P2P context [senderId=" + nodeId + ", msg=" + cacheMsg + ']');
http://git-wip-us.apache.org/repos/asf/ignite/blob/1649c532/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
index 2b696a5..59b2a68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryHandler.java
@@ -726,8 +726,13 @@ public class CacheContinuousQueryHandler<K, V> implements GridContinuousHandler
GridDeploymentInfo depInfo = e.deployInfo();
if (depInfo != null) {
- depMgr.p2pContext(nodeId, depInfo.classLoaderId(), depInfo.userVersion(), depInfo.deployMode(),
- depInfo.participants(), depInfo.localDeploymentOwner());
+ depMgr.p2pContext(
+ nodeId,
+ depInfo.classLoaderId(),
+ depInfo.userVersion(),
+ depInfo.deployMode(),
+ depInfo.participants()
+ );
}
}
[25/35] ignite git commit: ignite-3478
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/b72f362d/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java b/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java
deleted file mode 100644
index f008c32..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp3.java
+++ /dev/null
@@ -1,1713 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.GridAtomicLong;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.NotNull;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- *
- */
-public class MvccTestApp3 {
- /** */
- private static final boolean DEBUG_LOG = false;
-
- /** */
- private static final boolean SQL = false;
-
- public static void main1(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 2, true);
-
- Map<Object, Object> vals = cluster.sqlAll();
-
- System.out.println();
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)getData.get(i);
-
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main0(String[] args) throws Exception {
- final TestCluster cluster = new TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txRemoveTransfer(0, 1);
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (Map.Entry<Object, Object> e : getData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main(String[] args) throws Exception {
- final AtomicBoolean err = new AtomicBoolean();
-
- final int READ_THREADS = 4;
- final int UPDATE_THREADS = 4;
- final int ACCOUNTS = 50;
-
- final int START_VAL = 100000;
-
- for (int iter = 0; iter < 1000; iter++) {
- System.out.println("Iteration [readThreads=" + READ_THREADS +
- ", updateThreads=" + UPDATE_THREADS + ", accounts=" + ACCOUNTS + ", iter=" + iter + ']');
-
- final TestCluster cluster = new TestCluster(1);
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- List<Thread> threads = new ArrayList<>();
-
- Thread cleanupThread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("cleanup");
-
- try {
- while (!stop.get()) {
- cluster.cleanup();
-
- Thread.sleep(1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- threads.add(cleanupThread);
-
- cleanupThread.start();
-
- final boolean REMOVES = false;
-
- for (int i = 0; i < READ_THREADS; i++) {
- final int id = i;
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- try {
- Thread.currentThread().setName("read" + id);
-
- int cnt = 0;
-
- while (!stop.get()) {
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- cnt++;
-
- int sum = 0;
-
- if (REMOVES) {
- for (Map.Entry<Object, Object> e : qryData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
- else
- System.out.println("With null");
- }
- }
- else {
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- if (val == null) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("No value for key: " + i);
- }
-
- return;
- }
-
- sum += val;
- }
- }
-
- if (sum != ACCOUNTS * START_VAL) {
- if (stop.compareAndSet(false, true)) {
- err.set(true);
- stop.set(true);
-
- TestDebugLog.printAllAndExit("Invalid get sum: " + sum);
- }
- }
- }
-
- System.out.println("Get cnt: " + cnt);
- }
- catch (Throwable e) {
- e.printStackTrace();
-
- err.set(true);
- stop.set(true);
- }
- }
- });
-
- threads.add(thread);
-
- thread.start();
- }
-
- for (int i = 0; i < UPDATE_THREADS; i++) {
- final int id = i;
-
- Thread thread;
-
- if (REMOVES) {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (rnd.nextBoolean()) {
- cluster.txRemoveTransfer(id1, id2);
- }
- else
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
- else {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (id1 > id2) {
- int tmp = id1;
- id1 = id2;
- id2 = tmp;
- }
-
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
-
- threads.add(thread);
-
- thread.start();
- }
-
- long endTime = System.currentTimeMillis() + 2_000;
-
- while (!stop.get()) {
- Thread.sleep(1000);
-
- if (System.currentTimeMillis() >= endTime)
- break;
-
- //cluster.dumpMvccInfo();
- }
-
- stop.set(true);
-
- for (Thread thread : threads)
- thread.join();
-
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- System.out.println("Val " + val);
-
- if (val != null)
- sum += val;
- }
-
- System.out.println("Sum=" + sum + ", expSum=" + (ACCOUNTS * START_VAL));
-
- if (err.get()) {
- System.out.println("Error!");
-
- System.exit(1);
- }
-
-// cluster.dumpMvccInfo();
-//
-// System.out.println("Cleanup");
-//
-// cluster.cleanup();
-//
-// cluster.dumpMvccInfo();
-
- TestDebugLog.clear();
- }
- }
-
- /**
- *
- */
- static class TestCluster {
- /** */
- final List<Node> nodes = new ArrayList<>();
-
- /** */
- final Coordinator crd;
-
- /** */
- final AtomicLong txIdGen = new AtomicLong(10_000);
-
- TestCluster(int nodesNum) {
- crd = new Coordinator();
-
- for (int i = 0; i < nodesNum; i++)
- nodes.add(new Node(i));
- }
-
- void cleanup() {
- CoordinatorCounter cntr = crd.cleanupVersion();
-
- for (Node node : nodes)
- node.dataStore.cleanup(cntr);
- }
-
- void txPutAll(Map<Object, Object> data) {
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- for (Object key : data.keySet()) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), data.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId, cntr.cntr);
- }
-
- void txTransfer(Integer id1, Integer id2, boolean fromFirst) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(id1);
- keys.add(id2);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer curVal1 = (Integer)vals.get(id1);
- Integer curVal2 = (Integer)vals.get(id2);
-
- boolean update = false;
-
- Integer newVal1 = null;
- Integer newVal2 = null;
-
- if (curVal1 != null && curVal2 != null) {
- if (fromFirst) {
- if (curVal1 > 0) {
- update = true;
-
- newVal1 = curVal1 - 1;
- newVal2 = curVal2 + 1;
- }
- }
- else {
- if (curVal2 > 0) {
- update = true;
-
- newVal1 = curVal1 + 1;
- newVal2 = curVal2 - 1;
- }
- }
- }
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(id1, newVal1);
- newVals.put(id2, newVal2);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("update", txId, id1, newVal1, id2, newVal2, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId, cntr.cntr);
-
-// if (DEBUG_LOG)
-// TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- void txRemoveTransfer(Integer from, Integer to) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(from);
- keys.add(to);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer fromVal = (Integer)vals.get(from);
- Integer toVal = (Integer)vals.get(to);
-
- boolean update = fromVal != null && toVal != null;
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(from, null);
- newVals.put(to, fromVal + toVal);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("remove", txId, from, fromVal, to, toVal, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId, cntr.cntr);
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- public void dumpMvccInfo() {
- for (Node node : nodes) {
- int sql = node.dataStore.mvccSqlIdx.size();
-
- for (Map.Entry<Object, MvccValue> e : node.dataStore.mainIdx.entrySet()) {
- List<MvccValue> list = node.dataStore.mvccIdx.get(e.getKey());
-
- int size = 0;
-
- if (list != null) {
- synchronized (list) {
- size = list.size();
- }
- }
-
- System.out.println("Mvcc info [key=" + e.getKey() +
- ", val=" + e.getValue() +
- ", mvccVals=" + size +
- ", sqlVals=" + sql + ']');
- }
- }
- }
-
- public Map<Object, Object> sqlAll() {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Node node : nodes) {
- Map<Object, Object> nodeRes = node.dataStore.sqlQuery(qryVer);
-
- res.putAll(nodeRes);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sqlAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- public Map<Object, Object> getAll(Set<?> keys) {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Object val = node.dataStore.get(key, qryVer);
-
- res.put(key, val);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("getAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- private int nodeForKey(Object key) {
- return U.safeAbs(key.hashCode()) % nodes.size();
- }
- }
-
- /**
- *
- */
- static class Node {
- /** */
- final DataStore dataStore;
-
- /** */
- final int nodexIdx;
-
- public Node(int nodexIdx) {
- this.nodexIdx = nodexIdx;
-
- dataStore = new DataStore();
- }
-
- @Override public String toString() {
- return "Node [idx=" + nodexIdx + ']';
- }
- }
-
- /**
- *
- */
- static class Coordinator {
- /** */
- private final AtomicLong cntr = new AtomicLong(-1);
-
- /** */
- private final GridAtomicLong commitCntr = new GridAtomicLong(-1);
-
- /** */
- private final ConcurrentHashMap8<Long, QueryCounter> activeQueries = new ConcurrentHashMap8<>();
-
- /** */
- @GridToStringInclude
- private final ConcurrentHashMap8<TxId, Long> activeTxs = new ConcurrentHashMap8<>();
-
- CoordinatorCounter nextTxCounter(TxId txId) {
- long cur = cntr.get();
-
- activeTxs.put(txId, cur + 1);
-
- CoordinatorCounter newCtr = new CoordinatorCounter(cntr.incrementAndGet());
-
- return newCtr;
- }
-
- void txDone(TxId txId, long cntr) {
- Long rmvd = activeTxs.remove(txId);
-
- assert rmvd != null;
-
- commitCntr.setIfGreater(cntr);
- }
-
- private GridAtomicLong minActive0 = new GridAtomicLong(0);
-
- private Long minActive(Set<TxId> txs) {
- Long minActive = null;
-
- for (Map.Entry<TxId, Long> e : activeTxs.entrySet()) {
- if (txs != null)
- txs.add(e.getKey());
-
-// TxId val = e.getValue();
-//
-// while (val.cntr == -1)
-// Thread.yield();
-
- long cntr = e.getValue();
-
- if (minActive == null)
- minActive = cntr;
- else if (cntr < minActive)
- minActive = cntr;
- }
-
- if (minActive != null) {
- if (!minActive0.setIfGreater(minActive))
- return minActive0.get();
- }
-
- return minActive;
- }
-
- static class QueryCounter extends AtomicInteger {
- public QueryCounter(int initialValue) {
- super(initialValue);
- }
-
- boolean increment2() {
- for (;;) {
- int current = get();
- int next = current + 1;
-
- if (current == 0)
- return false;
-
- if (compareAndSet(current, next))
- return true;
- }
- }
- }
-
- private ReadWriteLock rwLock = new ReentrantReadWriteLock();
-
- MvccQueryVersion queryVersion() {
- rwLock.readLock().lock();
-
- long useCntr = commitCntr.get();
-
- Set<TxId> txs = new HashSet<>();
-
- Long minActive = minActive(txs);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- MvccQueryVersion qryVer = new MvccQueryVersion(new CoordinatorCounter(useCntr), txs);
-
- for (;;) {
- QueryCounter qryCnt = activeQueries.get(useCntr);
-
- if (qryCnt != null) {
- boolean inc = qryCnt.increment2();
-
- if (!inc) {
- activeQueries.remove(useCntr, qryCnt);
-
- continue;
- }
- }
- else {
- qryCnt = new QueryCounter(1);
-
- if (activeQueries.putIfAbsent(useCntr, qryCnt) != null)
- continue;
- }
-
- break;
- }
-
- rwLock.readLock().unlock();
-
- return qryVer;
- }
-
- void queryDone(CoordinatorCounter cntr) {
- AtomicInteger qryCnt = activeQueries.get(cntr.cntr);
-
- assert qryCnt != null : cntr.cntr;
-
- int left = qryCnt.decrementAndGet();
-
- assert left >= 0 : left;
-
- if (left == 0)
- activeQueries.remove(cntr.cntr, qryCnt);
- }
-
- CoordinatorCounter cleanupVersion() {
- rwLock.writeLock().lock();
-
- long useCntr = commitCntr.get();
-
- Long minActive = minActive(null);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- for (Long qryCntr : activeQueries.keySet()) {
- if (qryCntr <= useCntr)
- useCntr = qryCntr - 1;
- }
-
- rwLock.writeLock().unlock();
-
- return new CoordinatorCounter(useCntr);
- }
-
- @Override public String toString() {
- return S.toString(Coordinator.class, this);
- }
- }
-
- /**
- *
- */
- static class CoordinatorCounter implements Comparable<CoordinatorCounter> {
- /** */
- private final long topVer; // TODO
-
- /** */
- private final long cntr;
-
- CoordinatorCounter(long cntr) {
- this.topVer = 1;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(CoordinatorCounter o) {
- return Long.compare(cntr, o.cntr);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- CoordinatorCounter that = (CoordinatorCounter)o;
-
- return cntr == that.cntr;
- }
-
- @Override public int hashCode() {
- return (int)(cntr ^ (cntr >>> 32));
- }
-
- @Override public String toString() {
- return "Cntr [c=" + cntr + ']';
- }
- }
-
- /**
- *
- */
- static class MvccUpdateVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final TxId txId;
-
- /**
- * @param cntr
- */
- MvccUpdateVersion(CoordinatorCounter cntr, TxId txId) {
- assert cntr != null;
-
- this.cntr = cntr;
- this.txId = txId;
- }
-
- @Override public String toString() {
- return S.toString(MvccUpdateVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccQueryVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- MvccQueryVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
-
- @Override public String toString() {
- return S.toString(MvccQueryVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class TxId {
- /** */
- @GridToStringInclude
- final long id;
-
- TxId(long id) {
- this.id = id;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TxId txId = (TxId) o;
-
- return id == txId.id;
- }
-
- @Override public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override public String toString() {
- return S.toString(TxId.class, this);
- }
- }
-
- /**
- *
- */
- static class SqlKey implements Comparable<SqlKey> {
- /** */
- final Comparable key;
-
- /** */
- final Comparable val;
-
- /** */
- final CoordinatorCounter cntr;
-
- public SqlKey(Object key, Object val, CoordinatorCounter cntr) {
- this.key = (Comparable)key;
- this.val = (Comparable)val;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(@NotNull SqlKey o) {
- int cmp;
-
- if (val != null && o.val != null)
- cmp = val.compareTo(o.val);
- else {
- if (val != null)
- cmp = 1;
- else
- cmp = o.val == null ? 0 : -1;
- }
-
-
- if (cmp == 0) {
- cmp = key.compareTo(o.key);
-
- if (cmp == 0)
- cmp = cntr.compareTo(o.cntr);
- }
-
- return cmp;
- }
-
- @Override public String toString() {
- return "SqlKey [key=" + key + ", val=" + val + ']';
- }
- }
-
- /**
- *
- */
- static class DataStore {
- /** */
- private final ConcurrentHashMap<Object, ReentrantLock> locks = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, MvccValue> mainIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, List<MvccValue>> mvccIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentSkipListMap<SqlKey, MvccSqlValue> mvccSqlIdx = new ConcurrentSkipListMap<>();
-
- void cleanup(CoordinatorCounter cleanupCntr) {
- for (Map.Entry<Object, List<MvccValue>> e : mvccIdx.entrySet()) {
- lockEntry(e.getKey());
-
- try {
- List<MvccValue> list = e.getValue();
-
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (val.ver.cntr.compareTo(cleanupCntr) <= 0) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6_1("cleanup",
- e.getKey(), val.val, val.ver, cleanupCntr.cntr, null, null));
- }
-
- MvccValue prev;
-
- if (val.val != null)
- prev = mainIdx.put(e.getKey(), val);
- else
- prev = mainIdx.remove(e.getKey());
-
- if (prev != null) {
- SqlKey key = new SqlKey(e.getKey(), prev.val, prev.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
-
- for (int j = 0; j <= i; j++) {
- MvccValue rmvd = list.remove(0);
-
- assert rmvd != null;
-
- if (j != i || rmvd.val == null) {
- SqlKey key = new SqlKey(e.getKey(), rmvd.val, rmvd.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
- }
-
- if (list.isEmpty())
- mvccIdx.remove(e.getKey());
-
- break;
- }
- }
- }
- }
- finally {
- unlockEntry(e.getKey());
- }
- }
- }
-
- void lockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.lock();
- }
-
- void unlockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.unlock();
- }
-
- void updateEntry(Object key, Object val, MvccUpdateVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list == null) {
- Object old = mvccIdx.putIfAbsent(key, list = new ArrayList<>());
-
- assert old == null;
- }
-
- MvccValue prevVal = null;
-
- synchronized (list) {
- if (!list.isEmpty())
- prevVal = list.get(list.size() - 1);
-
- list.add(new MvccValue(val, ver));
- }
-
- if (prevVal == null)
- prevVal = mainIdx.get(key);
-
- if (prevVal != null) {
- SqlKey prevKey = new SqlKey(key, prevVal.val, prevVal.ver.cntr);
-
- MvccSqlValue old =
- mvccSqlIdx.put(prevKey, new MvccSqlValue(prevVal.val, prevVal.ver, ver));
-
- assert old != null;
- }
-
- mvccSqlIdx.put(new SqlKey(key, val, ver.cntr), new MvccSqlValue(val, ver, null));
- }
-
- Object lastValue(Object key) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- if (list.size() > 0)
- return list.get(list.size() - 1).val;
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- return val != null ? val.val : null;
- }
-
- Map<Object, Object> sqlQuery(MvccQueryVersion qryVer) {
- Map<Object, Object> res = new HashMap<>();
-
- for (Map.Entry<SqlKey, MvccSqlValue> e : mvccSqlIdx.entrySet()) {
- MvccSqlValue val = e.getValue();
-
- if (!versionVisible(val.ver, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sql skip mvcc val", e.getKey().key, val.val, val.ver));
- }
-
- continue;
- }
-
- MvccUpdateVersion newVer = val.newVer;
-
- if (newVer != null && versionVisible(newVer, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql skip mvcc val2", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- continue;
- }
-
- Object old = res.put(e.getKey().key, e.getValue().val);
-
- if (DEBUG_LOG) {
- //TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql get mvcc val", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- if (old != null) {
- TestDebugLog.printAllAndExit("Already has value for key [key=" + e.getKey().key +
- ", qryVer=" + qryVer +
- ", oldVal=" + old +
- ", newVal=" + e.getValue().val +
- ']');
- }
-
- assert old == null;
- }
-
- return res;
- }
-
- private boolean versionVisible(MvccUpdateVersion ver, MvccQueryVersion qryVer) {
- int cmp = ver.cntr.compareTo(qryVer.cntr);
-
- return cmp <= 0;// && !qryVer.activeTxs.contains(ver.txId);
- }
-
- Object get(Object key, MvccQueryVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (!versionVisible(val.ver, ver))
- continue;
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read mvcc val", key, val.val, val.ver));
- }
-
- return val.val;
- }
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- if (val != null) {
- int cmp = val.ver.cntr.compareTo(ver.cntr);
-
- if (DEBUG_LOG) {
- if (cmp > 0) {
- synchronized (TestDebugLog.msgs) {
- TestDebugLog.msgs.add(new TestDebugLog.Message("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver.cntr + ']'));
-
- TestDebugLog.printAllAndExit("Committed [key=" + key + ", ver=" + val.ver + ", qryVer=" + ver + ']');
- }
- }
- }
-
- assert cmp <= 0 : "Committed [ver=" + val.ver + ", qryVer=" + ver.cntr + ']';
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted val", key, val, val.ver));
- }
- else {
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted null", key, null, null));
- }
-
- return val != null ? val.val : null;
- }
-
- private ReentrantLock lock(Object key) {
- ReentrantLock e = locks.get(key);
-
- if (e == null) {
- ReentrantLock old = locks.putIfAbsent(key, e = new ReentrantLock());
-
- if (old != null)
- e = old;
- }
-
- return e;
- }
- }
-
- /**
- *
- */
- static class MvccValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- MvccValue(Object val, MvccUpdateVersion ver) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- }
-
- @Override public String toString() {
- return S.toString(MvccValue.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccSqlValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion newVer;
-
- MvccSqlValue(Object val, MvccUpdateVersion ver, MvccUpdateVersion newVer) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- this.newVer = newVer;
- }
-
- @Override public String toString() {
- return S.toString(MvccSqlValue.class, this);
- }
- }
-
- static void log(String msg) {
- System.out.println(Thread.currentThread() + ": " + msg);
- }
-
- static class TestDebugLog {
- /** */
- //static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(1_000_000));
- static final ConcurrentLinkedQueue<Object> msgs = new ConcurrentLinkedQueue<>();
-
-
-
- /** */
- private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
-
- static class Message {
- String thread = Thread.currentThread().getName();
-
- String msg;
-
- long ts = U.currentTimeMillis();
-
- public Message(String msg) {
- this.msg = msg;
- }
-
- public String toString() {
- return "Msg [msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg2 extends Message{
- Object v1;
- Object v2;
-
- public Msg2(String msg, Object v1, Object v2) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", msg=" + msg +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg3 extends Message{
- Object v1;
- Object v2;
- Object v3;
-
- public Msg3(String msg, Object v1, Object v2, Object v3) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg4 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
-
- public Msg4(String msg, Object v1, Object v2, Object v3, Object v4) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", v4=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg6 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", txId=" + v1 +
- ", id1=" + v2 +
- ", v1=" + v3 +
- ", id2=" + v4 +
- ", v2=" + v5 +
- ", cntr=" + v6 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
- static class Msg6_1 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6_1(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", key=" + v1 +
- ", val=" + v2 +
- ", ver=" + v3 +
- ", cleanupC=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class EntryMessage extends Message {
- Object key;
- Object val;
-
- public EntryMessage(Object key, Object val, String msg) {
- super(msg);
-
- this.key = key;
- this.val = val;
- }
-
- public String toString() {
- return "EntryMsg [key=" + key + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class PartMessage extends Message {
- int p;
- Object val;
-
- public PartMessage(int p, Object val, String msg) {
- super(msg);
-
- this.p = p;
- this.val = val;
- }
-
- public String toString() {
- return "PartMessage [p=" + p + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static final boolean out = false;
-
- public static void addMessage(String msg) {
- msgs.add(new Message(msg));
-
- if (out)
- System.out.println(msg);
- }
-
- public static void addEntryMessage(Object key, Object val, String msg) {
- if (key instanceof KeyCacheObject)
- key = ((KeyCacheObject)key).value(null, false);
-
- EntryMessage msg0 = new EntryMessage(key, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- public static void addPartMessage(int p, Object val, String msg) {
- PartMessage msg0 = new PartMessage(p, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- static void printAllAndExit(String msg) {
- System.out.println(msg);
-
- TestDebugLog.addMessage(msg);
-
- List<Object> msgs = TestDebugLog.printMessages(true, null);
-
- TestDebugLog.printMessages0(msgs, "test_debug_update.txt");
-
- TestDebugLog.printMessagesForThread(msgs, Thread.currentThread().getName(), "test_debug_thread.txt");
-
- System.exit(1);
- }
-
- public static void printMessagesForThread(List<Object> msgs0, String thread0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String thread = ((Message) msg).thread;
-
- if (thread.equals(thread0))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static void printMessages0(List<Object> msgs0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String msg0 = ((Message) msg).msg;
-
- if (msg0.equals("tx done") || msg0.equals("update") || msg0.equals("cleanup"))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static List<Object> printMessages(boolean file, Integer part) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (part != null && msg instanceof PartMessage) {
- if (((PartMessage) msg).p != part)
- continue;
- }
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0)
- System.out.println(msg);
- }
-
- return msgs0;
- }
-
- public static void printKeyMessages(boolean file, Object key) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- System.out.println(msg);
- }
- }
- }
-
- public static void clear() {
- msgs.clear();
- }
-
- public static void clearEntries() {
- for (Iterator it = msgs.iterator(); it.hasNext();) {
- Object msg = it.next();
-
- if (msg instanceof EntryMessage)
- it.remove();
- }
- }
-
- }}
[21/35] ignite git commit: IGNITE-5919 .NET: Add ReadKey at the end
of EntryProcessorExample
Posted by sb...@apache.org.
IGNITE-5919 .NET: Add ReadKey at the end of EntryProcessorExample
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9f662cc4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9f662cc4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9f662cc4
Branch: refs/heads/ignite-3479
Commit: 9f662cc4b524305af3126af49ccace7dab40216e
Parents: f120551
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Fri Sep 22 13:03:44 2017 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Sep 22 13:03:44 2017 +0300
----------------------------------------------------------------------
.../dotnet/Apache.Ignite.Core.Tests/Examples/Example.cs | 6 ++++++
.../Apache.Ignite.Examples/Datagrid/EntryProcessorExample.cs | 4 ++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9f662cc4/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/Example.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/Example.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/Example.cs
index e7a264d..9c3625f 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/Example.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Examples/Example.cs
@@ -56,8 +56,14 @@ namespace Apache.Ignite.Core.Tests.Examples
// Each example has a ReadKey at the end, which throws an exception in test environment.
if (ex.Message != "Cannot read keys when either application does not have a console or " +
"when console input has been redirected from a file. Try Console.Read.")
+ {
throw;
+ }
+
+ return;
}
+
+ throw new Exception("ReadKey missing at the end of the example.");
}
/// <summary>
http://git-wip-us.apache.org/repos/asf/ignite/blob/9f662cc4/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/EntryProcessorExample.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/EntryProcessorExample.cs b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/EntryProcessorExample.cs
index bd805c5..197a435 100644
--- a/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/EntryProcessorExample.cs
+++ b/modules/platforms/dotnet/examples/Apache.Ignite.Examples/Datagrid/EntryProcessorExample.cs
@@ -73,6 +73,10 @@ namespace Apache.Ignite.Examples.Datagrid
PrintCacheEntries(cache);
}
+
+ Console.WriteLine();
+ Console.WriteLine(">>> Example finished, press any key to exit ...");
+ Console.ReadKey();
}
/// <summary>
[12/35] ignite git commit: Updated github docs for .NET and C++
Posted by sb...@apache.org.
Updated github docs for .NET and C++
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/8cc6a596
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/8cc6a596
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/8cc6a596
Branch: refs/heads/ignite-3479
Commit: 8cc6a5964d784289224edd77243fc1f232b200a3
Parents: 271b32f
Author: Prachi Garg <pg...@gridgain.com>
Authored: Thu Sep 21 13:09:35 2017 -0700
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu Sep 21 13:09:35 2017 -0700
----------------------------------------------------------------------
README.md | 147 ++++++++++----------
modules/platforms/cpp/README.md | 215 ++++++++++++++++++++++++++++++
modules/platforms/dotnet/README.md | 229 +++++++++++++++++++++++---------
3 files changed, 462 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/8cc6a596/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index b4a976d..2306d16 100644
--- a/README.md
+++ b/README.md
@@ -6,7 +6,19 @@
<a href="http://ci.ignite.apache.org/viewType.html?buildTypeId=IgniteTests_IgniteBasic&branch_IgniteTests=%3Cdefault%3E"><img src="http://ci.ignite.apache.org/app/rest/builds/buildType:(id:IgniteTests_IgniteBasic)/statusIcon" /></a>
-[Apache Ignite][apache-homepage] is the <b>in-memory</b> computing platform
+
+## Getting Started
+
+For information on how to get started with Apache Ignite, please visit: [Getting Started][getting-started].
+
+## Full Documentation
+
+You can find the full Apache Ignite documentation here: [Full documentation][docs].
+
+
+## What is Apache Ignite?
+
+[Apache Ignite][apache-ignite-homepage] is the <b>in-memory</b> computing platform
that is <b>durable</b>, <b>strongly consistent</b>, and <b>highly available</b>
with powerful <b>SQL</b>, <b>key-value</b> and <b>processing</b> APIs.
@@ -66,20 +78,20 @@ You can view Apache Ignite as a collection of independent, well-integrated compo
scalability of your application.
Some of these components include:
-* Advanced Clustering
-* Data Grid
-* SQL Database
-* Compute Grid
-* Service Grid
-* Hadoop Accelerator
-* Spark Shared RDDs and SQL indexes
+* [Advanced Clustering](#advanced-clustering)
+* [Data Grid](#data-grid-jcache)
+* [SQL Database](#sql-database)
+* [Compute Grid](#compute-grid)
+* [Service Grid](#service-grid)
+* [Hadoop Accelerator](#hadoop-accelerator)
+* [Spark Shared RDDs and SQL indexes](#spark-shared-rdds)
-Below you’ll find a brief coverage for each some of them.
+Below you’ll find a brief explanation for each of them.
### Advanced Clustering
-Ignite nodes can automatically discover each other. This helps to scale the cluster when needed, without having to restart the whole cluster. Developers can also leverage from Ignite’s hybrid cloud support that allows establishing connection between private cloud and public clouds such as Amazon Web Services, providing them with best of both worlds.
+Ignite nodes can [automatically discover](https://apacheignite.readme.io/docs/cluster) each other. This helps to scale the cluster when needed, without having to restart the whole cluster. Developers can also leverage from Ignite’s hybrid cloud support that allows establishing connection between private cloud and public clouds such as Amazon Web Services, providing them with best of both worlds.
<p align="center">
<a href="https://apacheignite.readme.io/docs/cluster">
@@ -88,16 +100,17 @@ Ignite nodes can automatically discover each other. This helps to scale the clus
</p>
Apache Ignite can be deployed on:
-* AWS
-* Docker
-* Google Cloud
-* Kubernetes
-* Mesos
-* YARN
+* [AWS](https://apacheignite.readme.io/docs/aws-deployment)
+* [Docker](https://apacheignite.readme.io/docs/docker-deployment)
+* [Google Compute](https://apacheignite.readme.io/docs/google-compute-deployment)
+* [Kubernetes](https://apacheignite.readme.io/docs/kubernetes-deployment)
+* [Mesos](https://apacheignite.readme.io/docs/mesos-deployment)
+* [YARN](https://apacheignite.readme.io/docs/yarn-deployment)
+
### Data Grid (JCache)
-Ignite data grid is an in-memory distributed key-value store which can be viewed as a distributed partitioned hash map,
+[Ignite data grid](https://apacheignite.readme.io/docs/data-grid) is an in-memory distributed key-value store which can be viewed as a distributed partitioned hash map,
with every cluster node owning a portion of the overall data. This way the more cluster nodes we add, the more data we
can cache.
@@ -110,13 +123,13 @@ data sets using standard SQL syntax. Ignite supports standard SQL for querying i
<p align="center">
<a href="https://apacheignite.readme.io/docs/data-grid">
- <img src="https://ignite.apache.org/images/in-memory-data-grid.jpg" />
+ <img src="https://ignite.apache.org/images/data_grid.png" vspace="15" width="450px"/>
</a>
</p>
### SQL Database
-Apache Ignite incorporates distributed SQL database capabilities as a part of its platform. The database is horizontally
+Apache Ignite incorporates [distributed SQL database](https://apacheignite.readme.io/docs/distributed-sql) capabilities as a part of its platform. The database is horizontally
scalable, fault tolerant and SQL ANSI-99 compliant. It supports all SQL, DDL, and DML commands including SELECT, UPDATE,
INSERT, MERGE, and DELETE queries. It also provides support for a subset of DDL commands relevant for distributed
databases.
@@ -130,40 +143,40 @@ the Ignite JDBC or ODBC drivers. This provides a true cross-platform connectivit
<p align="center">
<a href="https://apacheignite.readme.io/docs/distributed-sql">
- <img src="https://ignite.apache.org/images/ignite-distributed-database.png" vspace="15" width="400px"/>
+ <img src="https://ignite.apache.org/images/sql_database.png" vspace="15" width="400px"/>
</a>
</p>
### Compute Grid
-Distributed computations are performed in parallel fashion to gain high performance, low latency, and linear scalability.
+[Distributed computations](https://apacheignite.readme.io/docs/compute-grid) are performed in parallel fashion to gain high performance, low latency, and linear scalability.
Ignite compute grid provides a set of simple APIs that allow users distribute computations and data processing across
multiple computers in the cluster. Distributed parallel processing is based on the ability to take any computation and
execute it on any set of cluster nodes and return the results back.
<p align="center">
<a href="https://apacheignite.readme.io/docs/compute-grid">
- <img src="https://ignite.apache.org/images/in_memory_compute.png" vspace="15"/>
+ <img src="https://ignite.apache.org/images/collocated_processing.png" vspace="15" width="400px"/>
</a>
</p>
We support these features, amongst others:
-* Distributed Closure Execution
-* MapReduce & ForkJoin Processing
-* Continuous Mapping
-* Clustered Executor Service
-* Per-Node Shared State
-* Collocation of Compute and Data
-* Load Balancing
-* Fault Tolerance
-* Job State Checkpointing
-* Job Scheduling
+* [Distributed Closure Execution](https://apacheignite.readme.io/docs/distributed-closures)
+* [MapReduce & ForkJoin Processing](https://apacheignite.readme.io/docs/compute-tasks)
+* [Continuous Mapping](https://apacheignite.readme.io/docs/continuous-mapping)
+* [Clustered Executor Service](https://apacheignite.readme.io/docs/executor-service)
+* [Per-Node Shared State](https://apacheignite.readme.io/docs/node-local-map)
+* [Collocation of Compute and Data](https://apacheignite.readme.io/docs/collocate-compute-and-data)
+* [Load Balancing](https://apacheignite.readme.io/docs/load-balancing)
+* [Fault Tolerance](https://apacheignite.readme.io/docs/fault-tolerance)
+* [Job State Checkpointing](https://apacheignite.readme.io/docs/checkpointing)
+* [Job Scheduling](https://apacheignite.readme.io/docs/job-scheduling)
### Service Grid
-Service Grid allows for deployments of arbitrary user-defined services on the cluster. You can implement and deploy any
+[Service Grid](https://apacheignite.readme.io/docs/service-grid) allows for deployments of arbitrary user-defined services on the cluster. You can implement and deploy any
service, such as custom counters, ID generators, hierarchical maps, etc.
Ignite allows you to control how many instances of your service should be deployed on each cluster node and will
@@ -171,13 +184,13 @@ automatically ensure proper deployment and fault tolerance of all the services.
<p align="center">
<a href="https://apacheignite.readme.io/docs/service-grid">
- <img src="https://ignite.apache.org/images/ignite_service.png" vspace="15"/>
+ <img src="https://ignite.apache.org/images/service_grid.png" vspace="15" width="400px"/>
</a>
</p>
### Ignite File System
-Ignite File System (IGFS) is an in-memory file system allowing work with files and directories over existing cache
+[Ignite File System](https://apacheignite-fs.readme.io/docs/in-memory-file-system) (IGFS) is an in-memory file system allowing work with files and directories over existing cache
infrastructure. IGFS can either work as purely in-memory file system, or delegate to another file system (e.g. various
Hadoop file system implementations) acting as a caching layer.
@@ -185,16 +198,17 @@ In addition, IGFS provides API to execute map-reduce tasks over file system data
### Distributed Data Structures
-Ignite supports complex data structures in a distributed fashion:
+Ignite supports complex [data structures](https://apacheignite.readme.io/docs/data-structures) in a distributed fashion:
-* Queues and sets: ordinary, bounded, collocated, non-collocated
-* Atomic types: `AtomicLong` and `AtomicReference`
-* `CountDownLatch`
-* ID Generators
+* [Queues and sets](https://apacheignite.readme.io/docs/queue-and-set): ordinary, bounded, collocated, non-collocated
+* [Atomic types](https://apacheignite.readme.io/docs/atomic-types): `AtomicLong` and `AtomicReference`
+* [CountDownLatch](https://apacheignite.readme.io/docs/countdownlatch)
+* [ID Generators](https://apacheignite.readme.io/docs/id-generator)
+* [Semaphore](https://apacheignite.readme.io/docs/distributed-semaphore)
### Distributed Messaging
-Distributed messaging allows for topic based cluster-wide communication between all nodes. Messages with a specified
+[Distributed messaging](https://apacheignite.readme.io/docs/messaging) allows for topic based cluster-wide communication between all nodes. Messages with a specified
message topic can be distributed to all or sub-group of nodes that have subscribed to that topic.
Ignite messaging is based on publish-subscribe paradigm where publishers and subscribers are connected together by a
@@ -202,15 +216,15 @@ common topic. When one of the nodes sends a message A for topic T, it is publish
### Distributed Events
-Distributed events allow applications to receive notifications when a variety of events occur in the distributed grid environment. You can automatically get notified for task executions, read, write or query operations occurring on local or remote nodes within the cluster.
+[Distributed events](https://apacheignite.readme.io/docs/events) allow applications to receive notifications when a variety of events occur in the distributed grid environment. You can automatically get notified for task executions, read, write or query operations occurring on local or remote nodes within the cluster.
### Hadoop Accelerator
-Our Hadoop Accelerator provides a set of components allowing for in-memory Hadoop job execution and file system operations.
+Our [Hadoop Accelerator](https://apacheignite-fs.readme.io/docs/hadoop-accelerator) provides a set of components allowing for in-memory Hadoop job execution and file system operations.
#### MapReduce
-An alternate high-performant implementation of job tracker which replaces standard Hadoop MapReduce. Use it to
+An alternate [high-performant](https://apacheignite-fs.readme.io/docs/map-reduce) implementation of job tracker which replaces standard Hadoop MapReduce. Use it to
boost your Hadoop MapReduce job execution performance.
<p align="center">
@@ -221,7 +235,7 @@ boost your Hadoop MapReduce job execution performance.
#### IGFS - In-Memory File System
-A Hadoop-compliant IGFS File System implementation over which Hadoop can run over in plug-n-play fashion and significantly reduce I/O and improve both, latency and throughput.
+A Hadoop-compliant [IGFS](https://apacheignite-fs.readme.io/docs/file-system) File System implementation over which Hadoop can run over in plug-n-play fashion and significantly reduce I/O and improve both, latency and throughput.
<p align="center">
<a href="https://apacheignite-fs.readme.io/docs/in-memory-file-system">
@@ -231,74 +245,76 @@ A Hadoop-compliant IGFS File System implementation over which Hadoop can run ove
#### Secondary File System
-An implementation of `SecondaryFileSystem`. This implementation can be injected into existing IGFS allowing for
+An implementation of [`SecondaryFileSystem`](https://apacheignite-fs.readme.io/docs/secondary-file-system). This implementation can be injected into existing IGFS allowing for
read-through and write-through behavior over any other Hadoop FileSystem implementation (e.g. HDFS). Use it if you
want your IGFS to become an in-memory caching layer over disk-based HDFS or any other Hadoop-compliant file system.
#### Supported Hadoop distributions
-* Apache Hadoop
-* Cloudera CDH
-* Hortonworks HDP
-* Apache BigTop
+* [Apache Hadoop](https://apacheignite-fs.readme.io/docs/installing-on-apache-hadoop)
+* [Cloudera CDH](https://apacheignite-fs.readme.io/docs/installing-on-cloudera-cdh)
+* [Hortonworks HDP](https://apacheignite-fs.readme.io/docs/installing-on-hortonworks-hdp)
+* [Apache Hive](https://apacheignite-fs.readme.io/docs/running-apache-hive-over-ignited-hadoop)
### Spark Shared RDDs
-Apache Ignite provides an implementation of Spark RDD abstraction which allows to easily share state in memory across Spark jobs. The main difference between native Spark `RDD` and `IgniteRDD` is that Ignite RDD provides a shared in-memory view on data across different Spark jobs, workers, or applications, while native Spark RDD cannot be seen by other Spark jobs or applications.
+Apache Ignite provides an implementation of Spark RDD abstraction which allows to easily share state in memory across Spark jobs.
+
+The main difference between native Spark `RDD` and `IgniteRDD` is that Ignite RDD provides a [shared in-memory](https://apacheignite-fs.readme.io/docs/ignite-for-spark) view on data across different Spark jobs, workers, or applications, while native Spark RDD cannot be seen by other Spark jobs or applications.
<p align="center">
<a href="https://apacheignite-fs.readme.io/docs/ignite-for-spark">
- <img src="https://ignite.apache.org/images/spark-ignite-rdd.png" height="400" vspace="15" />
+ <img src="https://ignite.apache.org/images/spark_integration.png" height="400" vspace="15" />
</a>
</p>
## Ignite Facts
-<b>Is Ignite a persistent or pure in-memory storage?</b><br/>
+#### Is Ignite a persistent or pure in-memory storage?
**Both**. Native persistence in Ignite can be turned on and off. This allows Ignite to store data sets bigger than can fit in the available memory. Essentially, the smaller operational data sets can be stored in-memory only, and larger data sets that do not fit in memory can be stored on disk, using memory as a caching layer for better performance.
[Read More](https://apacheignite.readme.io/docs/distributed-persistent-store)
-<b>Is Ignite a distributed database?</b><br/>
+#### Is Ignite a distributed database?
**Yes**. Data in Ignite is either *partitioned* or *replicated* across a cluster of multiple nodes. This provides scalability and adds resilience to the system. Ignite automatically controls how data is partitioned, however, users can plug in their own distribution (affinity) functions and collocate various pieces of data together for efficiency.
[Read More](https://apacheignite.readme.io/docs/distributed-sql)
-<b>Is Ignite a relational SQL database?</b><br/>
+#### Is Ignite a relational SQL database?
**Not fully**. Although Ignite aims to behave like any other relational SQL database, there are differences in how Ignite handles constraints and indexes. Ignite supports *primary* and *secondary* indexes, however, the *uniqueness* can only be enforced for the *primary* indexes. Ignite also does not support *foreign key* constraints.
Essentially, Ignite purposely does not support any constraints that would entail a cluster broadcast message for each update and significantly hurt performance and scalability of the system.
[Read More](https://apacheignite.readme.io/docs/indexes)
-<b>Is Ignite an in-memory database?</b><br/>
+#### Is Ignite an in-memory database?
**Yes**. Even though Ignite *durable memory* works well in-memory and on-disk, the disk persistence can be disabled and Ignite can act as a pure *in-memory database*.
[Read More](https://apacheignite.readme.io/docs/distributed-sql)
-<b>Is Ignite a transactional database?</b><br/>
+#### Is Ignite a transactional database?
**Not fully**. ACID Transactions are supported, but only at *key-value* API level. Ignite also supports *cross-partition* transactions, which means that transactions can span keys residing in different partitions on different servers.
At *SQL* level Ignite supports *atomic*, but not yet *transactional* consistency. Ignite community plans to implement SQL transactions in version 2.2.
[Read More](https://apacheignite.readme.io/docs/sql-queries#known-limitations)
-<b>Is Ignite a key-value store?</b><br/>
+#### Is Ignite a key-value store?
**Yes**. Ignite provides a feature rich key-value API, that is JCache (JSR-107) compliant and supports Java, C++, and .NET.
[Read More](https://apacheignite.readme.io/docs/data-grid)
-<b>Is Ignite an in-memory data grid?</b><br/>
+#### Is Ignite an in-memory data grid?
**Yes**. Ignite is a full-featured data grid, which can be used either in pure in-memory mode or with Ignite native persistence. It can also integrate with any 3rd party database, including any RDBMS or NoSQL store.
[Read More](https://apacheignite.readme.io/docs/data-grid)
-<b>What is durable memory?</b><br/>
+#### What is durable memory?
Ignite *durable memory* architecture allows Ignite to extend in-memory computing to disk. It is based on a paged-based off-heap memory allocator which becomes durable by persisting to the *write-ahead-log (WAL)* and, then, to main Ignite persistent storage. When persistence is disabled, durable memory acts like a pure in-memory storage.
[Read More](https://apacheignite.readme.io/docs/durable-memory)
-<b>What is collocated processing?</b><br/>
+#### What is collocated processing?
Ignite is a distributed system and, therefore, it is important to be able to collocate data with data and compute with data to avoid distributed data noise. Data collocation becomes especially important when performing distributed SQL joins. Ignite also supports sending user logic (functions, lambdas, etc.) directly to the nodes where the data resides and computing on the data locally.
[Read More](https://apacheignite.readme.io/docs/collocate-compute-and-data)
@@ -309,14 +325,7 @@ Ignite is a distributed system and, therefore, it is important to be able to col
<a href="modules/platforms/cpp">Ignite C++</a>
-## Getting Started
-
-For information on how to get started with Apache Ignite please visit: [Getting Started][getting-started].
-
-## Full Documentation
-
-You can find the full Apache Ignite documentation here: [Full documentation][docs].
-[apache-homepage]: https://ignite.apache.org/
+[apache-ignite-homepage]: https://ignite.apache.org/
[getting-started]: https://apacheignite.readme.io/docs/getting-started
[docs]: https://apacheignite.readme.io/docs
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/8cc6a596/modules/platforms/cpp/README.md
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/README.md b/modules/platforms/cpp/README.md
new file mode 100644
index 0000000..8b89004
--- /dev/null
+++ b/modules/platforms/cpp/README.md
@@ -0,0 +1,215 @@
+# Apache Ignite C++ In-Memory Computing Platform
+
+<a href="https://ignite.apache.org/"><img src="https://ignite.apache.org/images/logo3.png" hspace="20"/></a>
+
+[](https://gitter.im/apacheignite/ignite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+
+
+## Getting Started
+
+For information on how to get started with Apache Ignite, please visit: [Getting Started][getting-started].
+
+## Full Documentation
+
+You can find the full Apache Ignite documentation here: [Full documentation][docs].
+
+## What is Apache Ignite?
+
+[Apache Ignite][apache-ignite-homepage] is the <b>in-memory</b> computing platform
+that is <b>durable</b>, <b>strongly consistent</b>, and <b>highly available</b>
+with powerful <b>SQL</b>, <b>key-value</b> and <b>processing</b> APIs.
+
+<p align="center">
+ <a href="https://apacheignite-cpp.readme.io/docs/">
+ <img src="https://ignite.apache.org/images/durable-memory.png" width="900px"/>
+ </a>
+</p>
+
+## Durable Memory
+Ignite's durable memory component treats RAM not just as a caching layer but as a complete fully functional storage layer. This means that users can turn the persistence on and off as needed. If the persistence is off, then Ignite can act as a distributed **in-memory database** or **in-memory data grid**, depending on whether you prefer to use SQL or key-value APIs. If the persistence is turned on, then Ignite becomes a distributed, **horizontally scalable database** that guarantees full data consistency and is resilient to full cluster failures.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/durable-memory)
+
+## Ignite Persistence
+
+Ignite Native Persistence is a distributed, ACID, and SQL-compliant **disk store** that transparently integrates with Ignite's Durable Memory as an optional disk layer storing data and indexes on SSD, Flash, 3D XPoint, and other types of non-volatile storages.
+
+With the Ignite Persistence enabled, you no longer need to keep all the data and indexes in memory or warm it up after a node or cluster restart because the Durable Memory is tightly coupled with persistence and treats it as a secondary memory tier. This implies that if a subset of data or an index is missing in RAM, the Durable Memory will take it from the disk.
+
+<p align="center">
+ <a href="https://apacheignite.readme.io/docs/distributed-persistent-store">
+ <img src="https://ignite.apache.org/images/native_persistence.png" width="400px"/>
+ </a>
+</p>
+
+[Read More](https://apacheignite-cpp.readme.io/docs/ignite-persistence)
+
+## ACID Compliance
+Data stored in Ignite is ACID-compliant both in memory and on disk, making Ignite a **strongly consistent** system. Ignite transactions work across the network and can span multiple servers.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/transactions)
+
+## Complete SQL Support
+Ignite provides full support for SQL, DDL and DML, allowing users to interact with Ignite using pure SQL without writing any code. This means that users can create tables and indexes as well as insert, update, and query data using only SQL. Having such complete SQL support makes Ignite a one-of-a-kind **distributed SQL database**.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/sql-database)
+
+## Key-Value
+The in-memory data grid component in Ignite is a fully transactional **distributed key-value store** that can scale horizontally across 100s of servers in the cluster. When persistence is enabled, Ignite can also store more data than fits in memory and survive full cluster restarts.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/data-grid)
+
+## Collocated Processing
+Most traditional databases work in a client-server fashion, meaning that data must be brought to the client side for processing. This approach requires lots of data movement from servers to clients and generally does not scale. Ignite, on the other hand, allows for sending light-weight computations to the data, i.e. **collocating** computations with data. As a result, Ignite scales better and minimizes data movement.
+
+[Read More](https://apacheignite.readme.io/docs/collocate-compute-and-data)
+
+## Scalability and Durability
+Ignite is an elastic, horizontally scalable distributed system that supports adding and removing cluster nodes on demand. Ignite also allows for storing multiple copies of the data, making it resilient to partial cluster failures. If the persistence is enabled, then data stored in Ignite will also survive full cluster failures. Cluster restarts in Ignite can be very fast, as the data becomes operational instantaneously directly from disk. As a result, the data does not need to be preloaded in-memory to begin processing, and Ignite caches will lazily warm up resuming the in memory performance.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/cluster)
+
+## Ignite and Ignite С++
+* Ignite C++ is built on top of Ignite.
+* Ignite С++ starts the JVM in the same process and communicates with it via JNI.
+* .NET, C++ and Java nodes can join the same cluster, use the same caches, and interoperate using common binary protocol.
+* Java compute jobs can execute on any node (Java, .NET, C++).
+
+## Ignite Components
+
+You can view Apache Ignite as a collection of independent, well-integrated components geared to improve performance and
+ scalability of your application.
+
+Some of these components include:
+* [Advanced Clustering](#advanced-clustering)
+* [Data Grid](#data-grid-jcache)
+* [SQL Database](#sql-database)
+* [Compute Grid](#compute-grid)
+
+Below you’ll find a brief explanation for each of them:
+
+## Advanced Clustering
+
+Ignite nodes can [automatically discover](https://apacheignite-cpp.readme.io/docs/cluster) each other. This helps to scale the cluster when needed, without having to restart the whole cluster.
+Developers can also leverage Ignite’s hybrid cloud support that allows establishing connection between private cloud and public clouds
+such as Amazon Web Services, providing them with best of both worlds.
+
+<p align="center">
+ <a href="https://apacheignite-cpp.readme.io/docs/cluster">
+ <img src="https://ignite.apache.org/images/advanced-clustering.png" />
+ </a>
+</p>
+
+
+## Data Grid (Cache)
+
+[Ignite data grid](https://apacheignite-cpp.readme.io/docs/data-grid) is an in-memory distributed key-value store which can be viewed as a distributed partitioned `Dictionary`, with every cluster node
+owning a portion of the overall data. This way the more cluster nodes we add, the more data we can cache.
+
+Unlike other key-value stores, Ignite determines data locality using a pluggable hashing algorithm. Every client can determine which node a key
+belongs to by plugging it into a hashing function, without a need for any special mapping servers or name nodes.
+
+Ignite data grid supports local, replicated, and partitioned data sets and allows to freely cross query between these data sets using standard SQL and LINQ syntax.
+Ignite supports standard SQL and LINQ for querying in-memory data including support for distributed joins.
+
+<p align="center">
+ <a href="https://apacheignite-cpp.readme.io/docs/data-grid">
+ <img src="https://ignite.apache.org/images/data_grid.png" vspace="15" width="450px"/>
+ </a>
+</p>
+
+## SQL Database
+
+Apache Ignite incorporates [distributed SQL database](https://apacheignite-cpp.readme.io/docs/sql-database) capabilities as a part of its platform. The database is horizontally
+ scalable, fault tolerant and SQL ANSI-99 compliant. It supports all SQL, DDL, and DML commands including SELECT, UPDATE,
+ INSERT, MERGE, and DELETE queries. It also provides support for a subset of DDL commands relevant for distributed
+ databases.
+
+With Ignite Durable Memory architecture, data as well as indexes can be stored both in memory and, optionally, on disk.
+This allows executing distributed SQL operations across different memory layers achieving in-memory performance with the durability of disk.
+
+You can interact with Apache Ignite using the SQL language via natively developed APIs for Java, .NET and C++, or via
+the Ignite JDBC or ODBC drivers. This provides a true cross-platform connectivity from languages such as PHP, Ruby and more.
+
+
+<p align="center">
+ <a href="https://apacheignite-cpp.readme.io/docs/sql-database">
+ <img src="https://ignite.apache.org/images/sql_database.png" vspace="15" width="400px"/>
+ </a>
+</p>
+
+## Compute Grid
+
+[Distributed computations](https://apacheignite-cpp.readme.io/docs/compute-grid) are performed in parallel fashion to gain high performance, low latency, and linear scalability.
+Ignite compute grid provides a set of simple APIs that allow users distribute computations and data processing across multiple computers in the cluster.
+Distributed parallel processing is based on the ability to take any computation and execute it on any set of cluster nodes and return the results back.
+
+<p align="center">
+ <a href="https://apacheignite-cpp.readme.io/docs/compute-grid">
+ <img src="https://ignite.apache.org/images/collocated_processing.png" vspace="15" width="400px"/>
+ </a>
+</p>
+
+We support these features, amongst others:
+
+* [Distributed Closure Execution](https://apacheignite-cpp.readme.io/docs/distributed-closures)
+* [Load Balancing](https://apacheignite-cpp.readme.io/docs/load-balancing)
+* [Job Scheduling](https://apacheignite-cpp.readme.io/docs/job-scheduling)
+* [Fault Tolerance](https://apacheignite-cpp.readme.io/docs/fault-tolerance)
+
+
+## Ignite Facts
+
+#### Is Ignite a persistent or pure in-memory storage?
+**Both**. Native persistence in Ignite can be turned on and off. This allows Ignite to store data sets bigger than can fit in the available memory. Essentially, the smaller operational data sets can be stored in-memory only, and larger data sets that do not fit in memory can be stored on disk, using memory as a caching layer for better performance.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/ignite-persistence)
+
+#### Is Ignite a distributed database?
+**Yes**. Data in Ignite is either *partitioned* or *replicated* across a cluster of multiple nodes. This provides scalability and adds resilience to the system. Ignite automatically controls how data is partitioned, however, users can plug in their own distribution (affinity) functions and collocate various pieces of data together for efficiency.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/sql-database)
+
+#### Is Ignite a relational SQL database?
+**Not fully**. Although Ignite aims to behave like any other relational SQL database, there are differences in how Ignite handles constraints and indexes. Ignite supports *primary* and *secondary* indexes, however, the *uniqueness* can only be enforced for the *primary* indexes. Ignite also does not support *foreign key* constraints.
+
+Essentially, Ignite purposely does not support any constraints that would entail a cluster broadcast message for each update and significantly hurt performance and scalability of the system.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/sql-database)
+
+#### Is Ignite an in-memory database?
+**Yes**. Even though Ignite *durable memory* works well in-memory and on-disk, the disk persistence can be disabled and Ignite can act as a pure *in-memory database*.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/sql-database)
+
+#### Is Ignite a transactional database?
+**Not fully**. ACID Transactions are supported, but only at *key-value* API level. Ignite also supports *cross-partition* transactions, which means that transactions can span keys residing in different partitions on different servers.
+
+At *SQL* level Ignite supports *atomic*, but not yet *transactional* consistency. Ignite community plans to implement SQL transactions in version 2.4.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/sql-database)
+
+#### Is Ignite a key-value store?
+**Yes**. Ignite provides a feature rich key-value API, that is JCache (JSR-107) compliant and supports Java, C++, and .NET.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/data-grid)
+
+#### Is Ignite an in-memory data grid?
+**Yes**. Ignite is a full-featured data grid, which can be used either in pure in-memory mode or with Ignite native persistence. It can also integrate with any 3rd party database, including any RDBMS or NoSQL store.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/data-grid)
+
+#### What is durable memory?
+Ignite *durable memory* architecture allows Ignite to extend in-memory computing to disk. It is based on a paged-based off-heap memory allocator which becomes durable by persisting to the *write-ahead-log (WAL)* and, then, to main Ignite persistent storage. When persistence is disabled, durable memory acts like a pure in-memory storage.
+
+[Read More](https://apacheignite-cpp.readme.io/docs/durable-memory)
+
+#### What is collocated processing?
+Ignite is a distributed system and, therefore, it is important to be able to collocate data with data and compute with data to avoid distributed data noise. Data collocation becomes especially important when performing distributed SQL joins. Ignite also supports sending user logic (functions, lambdas, etc.) directly to the nodes where the data resides and computing on the data locally.
+
+[Read More](https://apacheignite.readme.io/docs/collocate-compute-and-data)
+
+
+[apache-ignite-homepage]: https://ignite.apache.org/
+[getting-started]: https://apacheignite-cpp.readme.io/docs/getting-started-1
+[docs]: https://apacheignite-cpp.readme.io/docs
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/8cc6a596/modules/platforms/dotnet/README.md
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/README.md b/modules/platforms/dotnet/README.md
index 1549f05..d2d6191 100644
--- a/modules/platforms/dotnet/README.md
+++ b/modules/platforms/dotnet/README.md
@@ -1,6 +1,6 @@
-# Apache Ignite.NET In-Memory Data Fabric
+# Apache Ignite.NET In-Memory Computing Platform
-<img src="https://ignite.apache.org/images/logo3.png" hspace="20" /><img src="https://ptupitsyn.github.io/images/net-framework.png" hspace="20" />
+<a href="https://ignite.apache.org/"><img src="https://ignite.apache.org/images/logo3.png" hspace="20"/></a><img src="https://ptupitsyn.github.io/images/net-framework.png" hspace="20" />
[](https://gitter.im/apacheignite/ignite?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -10,25 +10,108 @@
<a href="http://ci.ignite.apache.org/viewType.html?buildTypeId=Ignite20Tests_IgnitePlatformNet&branch_IgniteTests=%3Cdefault%3E"><img src="http://ci.ignite.apache.org/app/rest/builds/buildType:(id:Ignite20Tests_IgnitePlatformNet)/statusIcon" /></a>
-Apache Ignite In-Memory Data Fabric is designed to deliver uncompromised performance for a wide set of in-memory computing use cases from
-[high performance computing](https://ignite.apache.org/features.html), to the industry most advanced [data grid](https://ignite.apache.org/features.html),
-highly available [service grid](https://ignite.apache.org/features.html), and [streaming](https://ignite.apache.org/features.html).
+## Getting Started
+
+For information on how to get started with Apache Ignite, please visit: [Getting Started][getting-started].
+
+## Full Documentation
+
+You can find the full Apache Ignite documentation here: [Full documentation][docs].
+
+## What is Apache Ignite?
+
+[Apache Ignite][apache-ignite-homepage] is the <b>in-memory</b> computing platform
+that is <b>durable</b>, <b>strongly consistent</b>, and <b>highly available</b>
+with powerful <b>SQL</b>, <b>key-value</b> and <b>processing</b> APIs.
+
+<p align="center">
+ <a href="https://apacheignite-net.readme.io/docs/">
+ <img src="https://ignite.apache.org/images/durable-memory.png" width="900px"/>
+ </a>
+</p>
+
+## Durable Memory
+Ignite's durable memory component treats RAM not just as a caching layer but as a complete fully functional storage layer. This means that users can turn the persistence on and off as needed. If the persistence is off, then Ignite can act as a distributed **in-memory database** or **in-memory data grid**, depending on whether you prefer to use SQL or key-value APIs. If the persistence is turned on, then Ignite becomes a distributed, **horizontally scalable database** that guarantees full data consistency and is resilient to full cluster failures.
+
+[Read More](https://apacheignite-net.readme.io/docs/durable-memory)
+
+## Ignite Persistence
+
+Ignite Native Persistence is a distributed, ACID, and SQL-compliant **disk store** that transparently integrates with Ignite's Durable Memory as an optional disk layer storing data and indexes on SSD, Flash, 3D XPoint, and other types of non-volatile storages.
+
+With the Ignite Persistence enabled, you no longer need to keep all the data and indexes in memory or warm it up after a node or cluster restart because the Durable Memory is tightly coupled with persistence and treats it as a secondary memory tier. This implies that if a subset of data or an index is missing in RAM, the Durable Memory will take it from the disk.
+
+<p align="center">
+ <a href="https://apacheignite.readme.io/docs/distributed-persistent-store">
+ <img src="https://ignite.apache.org/images/native_persistence.png" width="400px"/>
+ </a>
+</p>
+
+[Read More](https://apacheignite-net.readme.io/docs/ignite-persistent-store)
+
+## ACID Compliance
+Data stored in Ignite is ACID-compliant both in memory and on disk, making Ignite a **strongly consistent** system. Ignite transactions work across the network and can span multiple servers.
+
+[Read More](https://apacheignite-net.readme.io/docs/transactions)
+
+## Complete SQL Support
+Ignite provides full support for SQL, DDL and DML, allowing users to interact with Ignite using pure SQL without writing any code. This means that users can create tables and indexes as well as insert, update, and query data using only SQL. Having such complete SQL support makes Ignite a one-of-a-kind **distributed SQL database**.
+
+[Read More](https://apacheignite-net.readme.io/docs/sql-database)
+
+## Key-Value
+The in-memory data grid component in Ignite is a fully transactional **distributed key-value store** that can scale horizontally across 100s of servers in the cluster. When persistence is enabled, Ignite can also store more data than fits in memory and survive full cluster restarts.
+
+[Read More](https://apacheignite-net.readme.io/docs/data-grid)
+
+## Collocated Processing
+Most traditional databases work in a client-server fashion, meaning that data must be brought to the client side for processing. This approach requires lots of data movement from servers to clients and generally does not scale. Ignite, on the other hand, allows for sending light-weight computations to the data, i.e. **collocating** computations with data. As a result, Ignite scales better and minimizes data movement.
+
+[Read More](https://apacheignite-net.readme.io/docs/colocate-compute-and-data)
+
+## Scalability and Durability
+Ignite is an elastic, horizontally scalable distributed system that supports adding and removing cluster nodes on demand. Ignite also allows for storing multiple copies of the data, making it resilient to partial cluster failures. If the persistence is enabled, then data stored in Ignite will also survive full cluster failures. Cluster restarts in Ignite can be very fast, as the data becomes operational instantaneously directly from disk. As a result, the data does not need to be preloaded in-memory to begin processing, and Ignite caches will lazily warm up resuming the in memory performance.
+
+[Read More](https://apacheignite-net.readme.io/docs/cluster)
+
+##Ignite and Ignite.NET
+
+* Ignite.NET is built on top of Ignite.
+* .NET starts the JVM in the same process and communicates with it via JNI & C++.
+* .NET, C++ and Java nodes can join the same cluster, use the same caches, and interoperate using common binary protocol.
+* Java compute jobs can execute on any node (Java, .NET, C++).
+* .NET compute jobs can only execute on .NET nodes.
+
+## Ignite Components
+
+You can view Apache Ignite as a collection of independent, well-integrated components geared to improve performance and
+ scalability of your application.
+
+Some of these components include:
+* [Advanced Clustering](#advanced-clustering)
+* [Data Grid](##data-grid-jcache)
+* [SQL Database](#sql-database)
+* [Compute Grid](#compute-grid)
+* [Service Grid](#service-grid)
+
+Below you’ll find a brief explanation for each of them:
## Advanced Clustering
-Ignite nodes can automatically discover each other. This helps to scale the cluster when needed, without having to restart the whole cluster.
-Developers can also leverage from Ignite’s hybrid cloud support that allows establishing connection between private cloud and public clouds
+Ignite nodes can [automatically discover](https://apacheignite-net.readme.io/docs/cluster) each other. This helps to scale the cluster when needed, without having to restart the whole cluster.
+Developers can also leverage Ignite’s hybrid cloud support that allows establishing connection between private cloud and public clouds
such as Amazon Web Services, providing them with best of both worlds.
<p align="center">
- <a href="https://apacheignite.readme.io/docs/cluster">
+ <a href="https://apacheignite-net.readme.io/docs/cluster">
<img src="https://ignite.apache.org/images/advanced-clustering.png" />
</a>
</p>
+
## Data Grid (Cache)
-Ignite data grid is an in-memory distributed key-value store which can be viewed as a distributed partitioned `Dictionary`, with every cluster node
+[Ignite data grid](https://apacheignite-net.readme.io/docs/data-grid) is an in-memory distributed key-value store which can be viewed as a distributed partitioned `Dictionary`, with every cluster node
owning a portion of the overall data. This way the more cluster nodes we add, the more data we can cache.
Unlike other key-value stores, Ignite determines data locality using a pluggable hashing algorithm. Every client can determine which node a key
@@ -38,87 +121,69 @@ Ignite data grid supports local, replicated, and partitioned data sets and allow
Ignite supports standard SQL and LINQ for querying in-memory data including support for distributed joins.
<p align="center">
- <a href="https://apacheignite.readme.io/docs/data-grid">
- <img src="https://ignite.apache.org/images/in-memory-data-grid.jpg" />
+ <a href="https://apacheignite-net.readme.io/docs/data-grid">
+ <img src="https://ignite.apache.org/images/data_grid.png" vspace="15" width="450px"/>
</a>
</p>
-Our data grid offers many features, some of which are:
-
-* Primary & Backup Copies.
-* Near Caches.
-* Cache queries and SQL/LINQ queries.
-* Continuous Queries.
-* Transactions.
-* Off-Heap Memory.
-* Affinity Collocation.
-* Persistent Store.
-* Automatic Persistence.
-* Data Loading.
-* Eviction and Expiry Policies.
-* Data Rebalancing
-* Topology Validation.
-
-## Streaming & CEP
+## SQL Database
-Ignite streaming allows to process continuous never-ending streams of data in scalable and fault-tolerant fashion. The rates at which data can be injected into Ignite can be very high and easily exceed millions of events per second on a moderately sized cluster.
+Apache Ignite incorporates [distributed SQL database](https://apacheignite-net.readme.io/docs/sql-database) capabilities as a part of its platform. The database is horizontally
+ scalable, fault tolerant and SQL ANSI-99 compliant. It supports all SQL, DDL, and DML commands including SELECT, UPDATE,
+ INSERT, MERGE, and DELETE queries. It also provides support for a subset of DDL commands relevant for distributed
+ databases.
-Real-time data is ingested via data streamers. We offer streamers for JMS 1.1, Apache Kafka, MQTT, Twitter, Apache Flume and Apache Camel already, and we keep adding new ones every release.
+With Ignite Durable Memory architecture, data as well as indexes can be stored both in memory and, optionally, on disk.
+This allows executing distributed SQL operations across different memory layers achieving in-memory performance with the durability of disk.
-<p align="center">
- <a href="https://apacheignite.readme.io/docs/streaming--cep">
- <img src="https://ignite.apache.org/images/data-streamers.png" />
- </a>
-</p>
+You can interact with Apache Ignite using the SQL language via natively developed APIs for Java, .NET and C++, or via
+the Ignite JDBC or ODBC drivers. This provides a true cross-platform connectivity from languages such as PHP, Ruby and more.
-The data can then be queried within sliding windows, if needed:
<p align="center">
- <a href="https://apacheignite.readme.io/docs/streaming--cep">
- <img src="https://ignite.apache.org/images/sliding-event-window.png" />
+ <a href="https://apacheignite-net.readme.io/docs/sql-database">
+ <img src="https://ignite.apache.org/images/sql_database.png" vspace="15" width="400px"/>
</a>
</p>
## Compute Grid
-Distributed computations are performed in parallel fashion to gain high performance, low latency, and linear scalability. Ignite compute grid provides a set of simple APIs that allow users distribute computations and data processing across multiple computers in the cluster. Distributed parallel processing is based on the ability to take any computation and execute it on any set of cluster nodes and return the results back.
+[Distributed computations](https://apacheignite-net.readme.io/docs/compute-grid) are performed in parallel fashion to gain high performance, low latency, and linear scalability.
+Ignite compute grid provides a set of simple APIs that allow users distribute computations and data processing across multiple computers in the cluster.
+Distributed parallel processing is based on the ability to take any computation and execute it on any set of cluster nodes and return the results back.
<p align="center">
- <a href="https://apacheignite.readme.io/docs/compute-grid">
- <img src="https://ignite.apache.org/images/in_memory_compute.png" />
+ <a href="https://apacheignite-net.readme.io/docs/compute-grid">
+ <img src="https://ignite.apache.org/images/collocated_processing.png" vspace="15" width="400px"/>
</a>
</p>
We support these features, amongst others:
-* Distributed Closure Execution.
-* MapReduce & ForkJoin Processing.
-* Clustered Executor Service.
-* Collocation of Compute and Data.
-* Load Balancing.
-* Fault Tolerance.
-* Job State Checkpointing.
-* Job Scheduling.
+* [Distributed Closure Execution](https://apacheignite-net.readme.io/docs/distributed-closures)
+* [MapReduce & ForkJoin Processing](https://apacheignite-net.readme.io/docs/mapreduce-forkjoin)
+* [Collocation of Compute and Data](https://apacheignite-net.readme.io/docs/colocate-compute-and-data)
+* [Fault Tolerance](https://apacheignite-net.readme.io/docs/fault-tolerance)
## Service Grid
-Service Grid allows for deployments of arbitrary user-defined services on the cluster. You can implement and deploy any service, such as custom counters, ID generators, hierarchical maps, etc.
+[Service Grid](https://apacheignite-net.readme.io/docs/service-grid) allows for deployments of arbitrary user-defined services on the cluster. You can implement and deploy any service, such as custom counters, ID generators, hierarchical maps, etc.
Ignite allows you to control how many instances of your service should be deployed on each cluster node and will automatically ensure proper deployment and fault tolerance of all the services.
<p align="center">
- <a href="https://apacheignite.readme.io/docs/service-grid">
- <img src="https://ignite.apache.org/images/ignite_service.png" vspace="15"/>
+ <a href="https://apacheignite-net.readme.io/docs/service-grid">
+ <img src="https://ignite.apache.org/images/service_grid.png" vspace="15" width="400px"/>
</a>
</p>
## Distributed Data Structures
-Ignite.NET supports complex data structures in a distributed fashion: `AtomicLong`, `AtomicReference`, `AtomicSequence`.
+Ignite.NET supports complex [data structures](https://apacheignite-net.readme.io/docs/atomic-types) in a distributed fashion: `AtomicLong`, `AtomicReference`, `AtomicSequence`.
## Distributed Messaging
-Distributed messaging allows for topic based cluster-wide communication between all nodes. Messages with a specified message topic can be distributed to
+[Distributed messaging](https://apacheignite-net.readme.io/docs/topic-based) allows for topic based cluster-wide communication between all nodes. Messages with a specified message topic can be distributed to
all or sub-group of nodes that have subscribed to that topic.
Ignite messaging is based on publish-subscribe paradigm where publishers and subscribers are connected together by a common topic.
@@ -126,7 +191,7 @@ When one of the nodes sends a message A for topic T, it is published on all node
## Distributed Events
-Distributed events allow applications to receive notifications when a variety of events occur in the distributed grid environment.
+[Distributed events](https://apacheignite-net.readme.io/docs/local-and-remote-events) allow applications to receive notifications when a variety of events occur in the distributed grid environment.
You can automatically get notified for task executions, read, write or query operations occurring on local or remote nodes within the cluster.
## ASP.NET Integration
@@ -137,15 +202,59 @@ Ignite.NET provides [Output Cache](https://apacheignite-net.readme.io/docs/aspne
Ignite.NET [Entity Framework Second Level Cache](https://apacheignite-net.readme.io/docs/entity-framework-second-level-cache) improves Entity Framework performance by storing query results in Ignite caches.
-## Getting Started
+## Ignite Facts
-For information on how to get started with Apache Ignite please visit: [Getting Started][getting-started].
+#### Is Ignite a persistent or pure in-memory storage?
+**Both**. Native persistence in Ignite can be turned on and off. This allows Ignite to store data sets bigger than can fit in the available memory. Essentially, the smaller operational data sets can be stored in-memory only, and larger data sets that do not fit in memory can be stored on disk, using memory as a caching layer for better performance.
-## Full Documentation
+[Read More](https://apacheignite-net.readme.io/docs/ignite-persistent-store)
+
+#### Is Ignite a distributed database?
+**Yes**. Data in Ignite is either *partitioned* or *replicated* across a cluster of multiple nodes. This provides scalability and adds resilience to the system. Ignite automatically controls how data is partitioned, however, users can plug in their own distribution (affinity) functions and collocate various pieces of data together for efficiency.
+
+[Read More](https://apacheignite-net.readme.io/docs/sql-database)
+
+#### Is Ignite a relational SQL database?
+**Not fully**. Although Ignite aims to behave like any other relational SQL database, there are differences in how Ignite handles constraints and indexes. Ignite supports *primary* and *secondary* indexes, however, the *uniqueness* can only be enforced for the *primary* indexes. Ignite also does not support *foreign key* constraints.
+
+Essentially, Ignite purposely does not support any constraints that would entail a cluster broadcast message for each update and significantly hurt performance and scalability of the system.
+
+[Read More](https://apacheignite-net.readme.io/docs/sql-database)
+
+#### Is Ignite an in-memory database?
+**Yes**. Even though Ignite *durable memory* works well in-memory and on-disk, the disk persistence can be disabled and Ignite can act as a pure *in-memory database*.
+
+[Read More](https://apacheignite-net.readme.io/docs/sql-database)
+
+#### Is Ignite a transactional database?
+**Not fully**. ACID Transactions are supported, but only at *key-value* API level. Ignite also supports *cross-partition* transactions, which means that transactions can span keys residing in different partitions on different servers.
+
+At *SQL* level Ignite supports *atomic*, but not yet *transactional* consistency. Ignite community plans to implement SQL transactions in version 2.4.
+
+[Read More](https://apacheignite-net.readme.io/docs/sql-database)
+
+#### Is Ignite a key-value store?
+**Yes**. Ignite provides a feature rich key-value API, that is JCache (JSR-107) compliant and supports Java, C++, and .NET.
+
+[Read More](https://apacheignite-net.readme.io/docs/data-grid)
+
+#### Is Ignite an in-memory data grid?
+**Yes**. Ignite is a full-featured data grid, which can be used either in pure in-memory mode or with Ignite native persistence. It can also integrate with any 3rd party database, including any RDBMS or NoSQL store.
+
+[Read More](https://apacheignite-net.readme.io/docs/data-grid)
+
+#### What is durable memory?
+Ignite *durable memory* architecture allows Ignite to extend in-memory computing to disk. It is based on a paged-based off-heap memory allocator which becomes durable by persisting to the *write-ahead-log (WAL)* and, then, to main Ignite persistent storage. When persistence is disabled, durable memory acts like a pure in-memory storage.
+
+[Read More](https://apacheignite-net.readme.io/docs/durable-memory)
+
+#### What is collocated processing?
+Ignite is a distributed system and, therefore, it is important to be able to collocate data with data and compute with data to avoid distributed data noise. Data collocation becomes especially important when performing distributed SQL joins. Ignite also supports sending user logic (functions, lambdas, etc.) directly to the nodes where the data resides and computing on the data locally.
+
+[Read More](https://apacheignite-net.readme.io/docs/colocate-compute-and-data)
-You can find the full Apache Ignite documentation here: [Full documentation][docs].
-[apache-homepage]: https://ignite.apache.org/
+[apache-ignite-homepage]: https://ignite.apache.org/
[getting-started]: https://apacheignite-net.readme.io/docs/getting-started-2
[docs]: https://apacheignite-net.readme.io/docs
[15/35] ignite git commit: Updated docs
Posted by sb...@apache.org.
Updated docs
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/155c55a5
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/155c55a5
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/155c55a5
Branch: refs/heads/ignite-3479
Commit: 155c55a590c0d9e9ab17c6394a5940ce81a41240
Parents: 78dfb12
Author: Denis Magda <dm...@gridgain.com>
Authored: Thu Sep 21 16:23:38 2017 -0700
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu Sep 21 16:23:38 2017 -0700
----------------------------------------------------------------------
modules/platforms/cpp/README.md | 7 +++++++
modules/platforms/dotnet/README.md | 10 ++++++++--
2 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/155c55a5/modules/platforms/cpp/README.md
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/README.md b/modules/platforms/cpp/README.md
index 272f2d4..f5fc3f9 100644
--- a/modules/platforms/cpp/README.md
+++ b/modules/platforms/cpp/README.md
@@ -210,6 +210,13 @@ Ignite is a distributed system and, therefore, it is important to be able to col
[Read More](https://apacheignite.readme.io/docs/collocate-compute-and-data)
+## Ignite On Other Platforms
+
+<a href="https://github.com/apache/ignite">Java</a>
+
+<a href="https://github.com/apache/ignite/tree/master/modules/platforms/dotnet">Ignite.Net</a>
+
+
[apache-ignite-homepage]: https://ignite.apache.org/
[getting-started]: https://apacheignite-cpp.readme.io/docs/getting-started-1
[docs]: https://apacheignite-cpp.readme.io/docs
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/155c55a5/modules/platforms/dotnet/README.md
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/README.md b/modules/platforms/dotnet/README.md
index 4487c00..68b639e 100644
--- a/modules/platforms/dotnet/README.md
+++ b/modules/platforms/dotnet/README.md
@@ -74,7 +74,7 @@ Ignite is an elastic, horizontally scalable distributed system that supports add
[Read More](https://apacheignite-net.readme.io/docs/cluster)
-##Ignite and Ignite.NET
+## Ignite and Ignite.NET
* Ignite.NET is built on top of Ignite.
* .NET starts the JVM in the same process and communicates with it via JNI & C++.
@@ -260,4 +260,10 @@ Ignite is a distributed system and, therefore, it is important to be able to col
## Contribute to Ignite.NET
-See [Ignite.NET Development](https://cwiki.apache.org/confluence/display/IGNITE/Ignite.NET+Development) on wiki.
\ No newline at end of file
+See [Ignite.NET Development](https://cwiki.apache.org/confluence/display/IGNITE/Ignite.NET+Development) on wiki.
+
+## Ignite On Other Platforms
+
+<a href="https://github.com/apache/ignite">Java</a>
+
+<a href="https://github.com/apache/ignite/tree/master/modules/platforms/cpp">Ignite C++</a>
\ No newline at end of file
[32/35] ignite git commit: ignite-3478
Posted by sb...@apache.org.
ignite-3478
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f6b01483
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f6b01483
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f6b01483
Branch: refs/heads/ignite-3479
Commit: f6b014834fce7a08eb4b2345edab54f5b3654408
Parents: e449460
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 14:00:09 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 14:00:09 2017 +0300
----------------------------------------------------------------------
.../dht/GridPartitionedGetFuture.java | 2 +-
.../mvcc/CacheCoordinatorsSharedManager.java | 35 +++++++++++++++-----
.../query/GridCacheDistributedQueryManager.java | 2 +-
.../cache/query/GridCacheQueryManager.java | 2 +-
4 files changed, 30 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6b01483/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
index a6978b5..9b7d733 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedGetFuture.java
@@ -249,7 +249,7 @@ public class GridPartitionedGetFuture<K, V> extends CacheDistributedGetFutureAda
if (mvccVer != null) {
assert mvccCrd != null;
- cctx.shared().coordinators().ackQueryDone(mvccCrd, mvccVer.counter());
+ cctx.shared().coordinators().ackQueryDone(mvccCrd, mvccVer);
}
cache().sendTtlUpdateRequest(expiryPlc);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6b01483/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
index 0f7e71e..d266bb3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/CacheCoordinatorsSharedManager.java
@@ -176,13 +176,26 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
/**
* @param crd Coordinator.
- * @param cntr Counter assigned to query.
+ * @param mvccVer Query version.
*/
- public void ackQueryDone(ClusterNode crd, long cntr) {
+ public void ackQueryDone(ClusterNode crd, MvccCoordinatorVersion mvccVer) {
try {
+ long trackCntr = mvccVer.counter();
+
+ MvccLongList txs = mvccVer.activeTransactions();
+
+ if (txs != null) {
+ for (int i = 0; i < txs.size(); i++) {
+ long txId = txs.get(i);
+
+ if (txId < trackCntr)
+ trackCntr = txId;
+ }
+ }
+
cctx.gridIO().sendToGridTopic(crd,
MSG_TOPIC,
- new CoordinatorQueryAckRequest(cntr),
+ new CoordinatorQueryAckRequest(trackCntr),
MSG_POLICY);
}
catch (ClusterTopologyCheckedException e) {
@@ -190,7 +203,7 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
log.debug("Failed to send query ack, node left [crd=" + crd + ']');
}
catch (IgniteCheckedException e) {
- U.error(log, "Failed to send query ack [crd=" + crd + ", cntr=" + cntr + ']', e);
+ U.error(log, "Failed to send query ack [crd=" + crd + ", cntr=" + mvccVer + ']', e);
}
}
@@ -519,15 +532,21 @@ public class CacheCoordinatorsSharedManager<K, V> extends GridCacheSharedManager
MvccCoordinatorVersionResponse res = new MvccCoordinatorVersionResponse();
- for (Long txVer : activeTxs.values())
+ Long trackCntr = mvccCntr;
+
+ for (Long txVer : activeTxs.values()) {
+ if (txVer < trackCntr)
+ trackCntr = txVer;
+
res.addTx(txVer);
+ }
- Integer queries = activeQueries.get(mvccCntr);
+ Integer queries = activeQueries.get(trackCntr);
if (queries != null)
- activeQueries.put(mvccCntr, queries + 1);
+ activeQueries.put(trackCntr, queries + 1);
else
- activeQueries.put(mvccCntr, 1);
+ activeQueries.put(trackCntr, 1);
res.init(futId, crdVer, mvccCntr, COUNTER_NA);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6b01483/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
index ffb49e0..3433b4f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java
@@ -585,7 +585,7 @@ public class GridCacheDistributedQueryManager<K, V> extends GridCacheQueryManage
cctx.io().removeOrderedHandler(false, topic);
if (mvccCrd != null)
- cctx.shared().coordinators().ackQueryDone(mvccCrd, mvccVer.counter());
+ cctx.shared().coordinators().ackQueryDone(mvccCrd, mvccVer);
}
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6b01483/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
index 075e492..b711a80 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java
@@ -3004,7 +3004,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte
/** {@inheritDoc} */
@Override protected void onClose() {
if (mvccCrd != null)
- dht.context().shared().coordinators().ackQueryDone(mvccCrd, mvccVer.counter());
+ dht.context().shared().coordinators().ackQueryDone(mvccCrd, mvccVer);
if (expiryPlc != null && dht != null) {
dht.sendTtlUpdateRequest(expiryPlc);
[16/35] ignite git commit: IGNITE-6476 Fixed "No suitable driver
found" in demo mode.
Posted by sb...@apache.org.
IGNITE-6476 Fixed "No suitable driver found" in demo mode.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/27295f23
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/27295f23
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/27295f23
Branch: refs/heads/ignite-3479
Commit: 27295f238ffe9e7d81db955242fbd6c826717386
Parents: 155c55a
Author: Alexey Kuznetsov <ak...@apache.org>
Authored: Fri Sep 22 11:30:12 2017 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Sep 22 11:30:12 2017 +0700
----------------------------------------------------------------------
.../org/apache/ignite/console/demo/AgentMetadataDemo.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/27295f23/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentMetadataDemo.java
----------------------------------------------------------------------
diff --git a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentMetadataDemo.java b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentMetadataDemo.java
index 4683dd8..b017fb0 100644
--- a/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentMetadataDemo.java
+++ b/modules/web-console/web-agent/src/main/java/org/apache/ignite/console/demo/AgentMetadataDemo.java
@@ -58,6 +58,8 @@ public class AgentMetadataDemo {
log.info("DEMO: Prepare in-memory H2 database...");
try {
+ Class.forName("org.h2.Driver");
+
Connection conn = DriverManager.getConnection("jdbc:h2:mem:demo-db;DB_CLOSE_DELAY=-1", "sa", "");
File sqlScript = resolvePath("demo/db-init.sql");
@@ -75,6 +77,11 @@ public class AgentMetadataDemo {
log.info("DEMO: JDBC URL for test drive metadata load: jdbc:h2:mem:demo-db");
}
+ catch (ClassNotFoundException e) {
+ log.error("DEMO: Failed to load H2 driver!", e);
+
+ throw new SQLException("Failed to load H2 driver", e);
+ }
catch (SQLException e) {
log.error("DEMO: Failed to start test drive for metadata!", e);
[31/35] ignite git commit: IGNITE-6466 Fixed testGetForInitialWrite -
Fixes #2721.
Posted by sb...@apache.org.
IGNITE-6466 Fixed testGetForInitialWrite - Fixes #2721.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1f2f8724
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1f2f8724
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1f2f8724
Branch: refs/heads/ignite-3479
Commit: 1f2f87243953ab582b1cb8a6b1098cf817215868
Parents: 05eba9a
Author: dpavlov <dp...@gridgain.com>
Authored: Fri Sep 22 13:47:14 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 22 13:47:14 2017 +0300
----------------------------------------------------------------------
.../processors/cache/persistence/tree/io/DataPageIO.java | 6 +++---
.../internal/processors/cache/persistence/tree/io/PageIO.java | 2 +-
.../IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java | 4 +++-
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f2f8724/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
index e6876a1..628ff38 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/DataPageIO.java
@@ -68,8 +68,8 @@ public class DataPageIO extends PageIO {
/** */
private static final int FIRST_ENTRY_OFF = INDIRECT_CNT_OFF + 1;
- /** */
- private static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
+ /** Offset of items (internal page pointers) within data page */
+ public static final int ITEMS_OFF = FIRST_ENTRY_OFF + 2;
/** */
private static final int ITEM_SIZE = 2;
@@ -1445,7 +1445,7 @@ public class DataPageIO extends PageIO {
/** {@inheritDoc} */
@Override protected void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException {
sb.a("DataPageIO [\n");
- printPageLayout(addr, 0, sb);
+ printPageLayout(addr, pageSize, sb);
sb.a("\n]");
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f2f8724/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
index 11250ef..60b1aaf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/io/PageIO.java
@@ -557,7 +557,7 @@ public abstract class PageIO {
/**
* @param addr Address.
- * @param pageSize
+ * @param pageSize Page size.
* @param sb Sb.
*/
protected abstract void printPage(long addr, int pageSize, GridStringBuilder sb) throws IgniteCheckedException ;
http://git-wip-us.apache.org/repos/asf/ignite/blob/1f2f8724/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
index 297e69f..2278981 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/file/IgnitePdsCheckpointSimulationWithRealCpDisabledTest.java
@@ -235,8 +235,10 @@ public class IgnitePdsCheckpointSimulationWithRealCpDisabledTest extends GridCom
try {
DataPageIO.VERSIONS.latest().initNewPage(pageAddr, fullId.pageId(), mem.pageSize());
- for (int i = PageIO.COMMON_HEADER_END; i < mem.pageSize(); i++)
+ for (int i = PageIO.COMMON_HEADER_END + DataPageIO.ITEMS_OFF; i < mem.pageSize(); i++)
PageUtils.putByte(pageAddr, i, (byte)0xAB);
+
+ PageIO.printPage(pageAddr, mem.pageSize());
}
finally {
mem.writeUnlock(fullId.groupId(), fullId.pageId(), page, null, true);
[07/35] ignite git commit: IGNITE-6316: SQL: fixed CacheConfiguration
persistence logic for DDL operations. This closes #2701.
Posted by sb...@apache.org.
IGNITE-6316: SQL: fixed CacheConfiguration persistence logic for DDL operations. This closes #2701.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/37b4b1d1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/37b4b1d1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/37b4b1d1
Branch: refs/heads/ignite-3479
Commit: 37b4b1d1444e849ef1c74201f2d69e7a250d6c2d
Parents: f8ae8f9
Author: Alexander Paschenko <al...@gmail.com>
Authored: Thu Sep 21 18:51:18 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Thu Sep 21 18:51:18 2017 +0300
----------------------------------------------------------------------
.../pagemem/store/IgnitePageStoreManager.java | 4 +-
.../cache/CacheAffinitySharedManager.java | 4 +-
.../persistence/file/FilePageStoreManager.java | 7 +-
.../processors/query/GridQueryProcessor.java | 30 ++-
.../pagemem/NoOpPageStoreManager.java | 2 +-
.../IgnitePersistentStoreSchemaLoadTest.java | 237 ++++++++++++-------
6 files changed, 184 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/37b4b1d1/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
index eaa85ad..64c5927 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/store/IgnitePageStoreManager.java
@@ -20,7 +20,6 @@ package org.apache.ignite.internal.pagemem.store;
import java.nio.ByteBuffer;
import java.util.Map;
import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.internal.pagemem.PageMemory;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheGroupDescriptor;
@@ -185,9 +184,10 @@ public interface IgnitePageStoreManager extends GridCacheSharedManager, IgniteCh
/**
* @param cacheData Cache configuration.
+ * @param overwrite Whether stored configuration should be overwritten if it exists.
* @throws IgniteCheckedException If failed.
*/
- public void storeCacheData(StoredCacheData cacheData) throws IgniteCheckedException;
+ public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException;
/**
* @param grpId Cache group ID.
* @return {@code True} if index store for given cache group existed before node started.
http://git-wip-us.apache.org/repos/asf/ignite/blob/37b4b1d1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
index beb7a7d..741e204 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/CacheAffinitySharedManager.java
@@ -2596,8 +2596,8 @@ public class CacheAffinitySharedManager<K, V> extends GridCacheSharedManagerAdap
if (cctx.pageStore() != null && cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) {
try {
cctx.pageStore().storeCacheData(
- new StoredCacheData(cfg)
- );
+ new StoredCacheData(cfg),
+ false);
}
catch (IgniteCheckedException e) {
U.error(log(), "Error while saving cache configuration on disk, cfg = " + cfg, e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/37b4b1d1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
index d60151a..b3eb74a 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStoreManager.java
@@ -207,9 +207,7 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
}
/** {@inheritDoc} */
- @Override public void storeCacheData(
- StoredCacheData cacheData
- ) throws IgniteCheckedException {
+ @Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException {
File cacheWorkDir = cacheWorkDirectory(cacheData.config());
File file;
@@ -222,10 +220,11 @@ public class FilePageStoreManager extends GridCacheSharedManagerAdapter implemen
else
file = new File(cacheWorkDir, CACHE_DATA_FILENAME);
- if (!file.exists() || file.length() == 0) {
+ if (overwrite || !file.exists() || file.length() == 0) {
try {
file.createNewFile();
+ // Pre-existing file will be truncated upon stream open.
try (OutputStream stream = new BufferedOutputStream(new FileOutputStream(file))) {
marshaller.marshal(cacheData, stream);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/37b4b1d1/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
index e791101..51d3411 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/query/GridQueryProcessor.java
@@ -64,7 +64,9 @@ import org.apache.ignite.internal.processors.cache.CacheObjectContext;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.GridCacheAdapter;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.StoredCacheData;
import org.apache.ignite.internal.processors.cache.query.CacheQueryFuture;
import org.apache.ignite.internal.processors.cache.query.CacheQueryType;
import org.apache.ignite.internal.processors.cache.query.GridCacheQueryType;
@@ -503,8 +505,11 @@ public class GridQueryProcessor extends GridProcessorAdapter {
if (!msg.hasError()) {
DynamicCacheDescriptor cacheDesc = ctx.cache().cacheDescriptor(msg.operation().cacheName());
- if (cacheDesc != null && F.eq(cacheDesc.deploymentId(), proposeMsg.deploymentId()))
+ if (cacheDesc != null && F.eq(cacheDesc.deploymentId(), proposeMsg.deploymentId())) {
cacheDesc.schemaChangeFinish(msg);
+
+ saveCacheConfiguration(cacheDesc);
+ }
}
// Propose message will be used from exchange thread to
@@ -2504,6 +2509,29 @@ private IgniteInternalFuture<Object> rebuildIndexesFromHash(@Nullable final Stri
", sndNodeId=" + msg.senderNodeId() + ']');
}
}
+ /**
+ * @param desc cache descriptor.
+ */
+ private void saveCacheConfiguration(DynamicCacheDescriptor desc) {
+ GridCacheSharedContext cctx = ctx.cache().context();
+
+ if (cctx.pageStore() != null && cctx.database().persistenceEnabled() && !cctx.kernalContext().clientNode()) {
+ CacheConfiguration cfg = desc.cacheConfiguration();
+
+ try {
+ StoredCacheData data = new StoredCacheData(cfg);
+
+ if (desc.schema() != null)
+ data.queryEntities(desc.schema().entities());
+
+ cctx.pageStore().storeCacheData(data, true);
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Error while saving cache configuration on disk, cfg = " + cfg, e);
+ }
+ }
+ }
+
/**
* Unwind pending messages for particular operation.
http://git-wip-us.apache.org/repos/asf/ignite/blob/37b4b1d1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
index 8fc2bdb..40887e8c 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpPageStoreManager.java
@@ -172,7 +172,7 @@ public class NoOpPageStoreManager implements IgnitePageStoreManager {
}
/** {@inheritDoc} */
- @Override public void storeCacheData(StoredCacheData cacheData) throws IgniteCheckedException {
+ @Override public void storeCacheData(StoredCacheData cacheData, boolean overwrite) throws IgniteCheckedException {
// No-op.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/37b4b1d1/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
index 349f062..b4c08b2 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/database/IgnitePersistentStoreSchemaLoadTest.java
@@ -18,30 +18,27 @@
package org.apache.ignite.internal.processors.database;
import java.io.Serializable;
-import java.util.Arrays;
-import java.util.LinkedHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.CountDownLatch;
+import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.cache.CacheAtomicityMode;
import org.apache.ignite.cache.CacheRebalanceMode;
import org.apache.ignite.cache.QueryEntity;
-import org.apache.ignite.cache.QueryIndex;
-import org.apache.ignite.cache.QueryIndexType;
+import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.annotations.QuerySqlField;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.PersistentStoreConfiguration;
import org.apache.ignite.internal.IgniteEx;
-import org.apache.ignite.internal.IgniteInterruptedCheckedException;
import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor;
import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener;
import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager;
-import org.apache.ignite.internal.util.lang.GridAbsPredicate;
+import org.apache.ignite.internal.processors.query.QuerySchema;
+import org.apache.ignite.internal.processors.query.QueryUtils;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
-import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK;
@@ -53,21 +50,17 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest
/** */
private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
- /** Index name. */
- private static final String IDX_NAME = "my_idx";
-
/** Cache name. */
private static final String TMPL_NAME = "test_cache*";
/** Table name. */
private static final String TBL_NAME = Person.class.getSimpleName();
- /** Schema name. */
- private static final String SCHEMA_NAME = "PUBLIC";
-
- /** Cache name. */
- private static final String CACHE_NAME = TBL_NAME;
+ /** Name of the cache created with {@code CREATE TABLE}. */
+ private static final String SQL_CACHE_NAME = QueryUtils.createTableCacheName(QueryUtils.DFLT_SCHEMA, TBL_NAME);
+ /** Name of the cache created upon cluster start. */
+ private static final String STATIC_CACHE_NAME = TBL_NAME;
/** {@inheritDoc} */
@Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
@@ -83,9 +76,31 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest
cfg.setPersistentStoreConfiguration(pCfg);
+ cfg.setActiveOnStart(true);
+
return cfg;
}
+ /**
+ * Create node configuration with a cache pre-configured.
+ * @param gridName Node name.
+ * @return Node configuration with a cache pre-configured.
+ * @throws Exception if failed.
+ */
+ @SuppressWarnings("unchecked")
+ private IgniteConfiguration getConfigurationWithStaticCache(String gridName) throws Exception {
+ IgniteConfiguration cfg = getConfiguration(gridName);
+
+ CacheConfiguration ccfg = cacheCfg(STATIC_CACHE_NAME);
+
+ ccfg.setIndexedTypes(Integer.class, Person.class);
+ ccfg.setSqlEscapeAll(true);
+
+ cfg.setCacheConfiguration(ccfg);
+
+ return optimize(cfg);
+ }
+
/** */
private CacheConfiguration cacheCfg(String name) {
CacheConfiguration<?, ?> cfg = new CacheConfiguration<>();
@@ -99,20 +114,6 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest
return cfg;
}
- /** */
- private QueryEntity getEntity() {
- LinkedHashMap<String, String> fields = new LinkedHashMap<>();
-
- fields.put("id", Integer.class.getName());
- fields.put("name", String.class.getName());
-
- QueryEntity entity = new QueryEntity(Integer.class.getName(), Person.class.getName());
- entity.setFields(fields);
- entity.setTableName(TBL_NAME);
-
- return entity;
- }
-
/** {@inheritDoc} */
@Override protected void beforeTest() throws Exception {
System.setProperty(IGNITE_SKIP_CONFIGURATION_CONSISTENCY_CHECK, "true");
@@ -132,98 +133,112 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest
}
/** */
- public void testPersistIndex() throws Exception {
- IgniteEx ig0 = startGrid(0);
- startGrid(1);
+ public void testDynamicSchemaChangesPersistence() throws Exception {
+ checkSchemaStateAfterNodeRestart(false);
+ }
- final AtomicInteger cnt = new AtomicInteger();
+ /** */
+ public void testDynamicSchemaChangesPersistenceWithAliveCluster() throws Exception {
+ checkSchemaStateAfterNodeRestart(true);
+ }
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
+ /** */
+ @SuppressWarnings("unchecked")
+ public void testDynamicSchemaChangesPersistenceWithStaticCache() throws Exception {
+ IgniteEx node = startGrid(getConfigurationWithStaticCache(getTestIgniteInstanceName(0)));
- db.addCheckpointListener(new DbCheckpointListener() {
- @Override public void onCheckpointBegin(Context context) {
- cnt.incrementAndGet();
- }
- });
+ node.active(true);
- QueryIndex idx = new QueryIndex("name");
+ IgniteCache cache = node.cache(STATIC_CACHE_NAME);
- idx.setName(IDX_NAME);
+ assertNotNull(cache);
- ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null,
- null, 1, true);
+ CountDownLatch cnt = checkpointLatch(node);
- assert indexCnt(ig0, CACHE_NAME) == 0;
+ assertEquals(0, indexCnt(node, STATIC_CACHE_NAME));
- ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get();
+ makeDynamicSchemaChanges(node, STATIC_CACHE_NAME);
- assert indexCnt(ig0, CACHE_NAME) == 1;
+ checkDynamicSchemaChanges(node, STATIC_CACHE_NAME);
- waitForCheckpoint(cnt);
+ cnt.await();
- stopGrid(1);
+ stopGrid(0);
- IgniteEx ig1 = startGrid(1);
+ // Restarting with no-cache configuration - otherwise stored configurations
+ // will be ignored due to cache names duplication.
+ node = startGrid(0);
- assert indexCnt(ig1, CACHE_NAME) == 1;
- }
+ node.active(true);
- /** */
- public void testPersistCompositeIndex() throws Exception {
- IgniteEx ig0 = startGrid(0);
- startGrid(1);
+ checkDynamicSchemaChanges(node, STATIC_CACHE_NAME);
+ }
- final AtomicInteger cnt = new AtomicInteger();
+ /**
+ * Perform test with cache created with {@code CREATE TABLE}.
+ * @param aliveCluster Whether there should remain an alive node when tested node is restarted.
+ * @throws Exception if failed.
+ */
+ private void checkSchemaStateAfterNodeRestart(boolean aliveCluster) throws Exception {
+ IgniteEx node = startGrid(0);
- GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)ig0.context().cache().context().database();
+ node.active(true);
- db.addCheckpointListener(new DbCheckpointListener() {
- @Override public void onCheckpointBegin(Context context) {
- cnt.incrementAndGet();
- }
- });
+ if (aliveCluster)
+ startGrid(1);
- ig0.context().query().dynamicTableCreate(SCHEMA_NAME, getEntity(), TMPL_NAME, null, null, null, null, 1, true);
+ CountDownLatch cnt = checkpointLatch(node);
- assert indexCnt(ig0, CACHE_NAME) == 0;
+ node.context().query().querySqlFieldsNoCache(
+ new SqlFieldsQuery("create table \"Person\" (\"id\" int primary key, \"name\" varchar)"), false).getAll();
- QueryIndex idx = new QueryIndex(Arrays.asList("id", "name"), QueryIndexType.SORTED);
+ assertEquals(0, indexCnt(node, SQL_CACHE_NAME));
- idx.setName(IDX_NAME);
+ makeDynamicSchemaChanges(node, QueryUtils.DFLT_SCHEMA);
- ig0.context().query().dynamicIndexCreate(CACHE_NAME, SCHEMA_NAME, TBL_NAME, idx, false).get();
+ checkDynamicSchemaChanges(node, SQL_CACHE_NAME);
- assert indexCnt(ig0, CACHE_NAME) == 1;
+ cnt.await();
- waitForCheckpoint(cnt);
+ stopGrid(0);
- stopGrid(1);
+ node = startGrid(0);
- IgniteEx ig1 = startGrid(1);
+ node.active(true);
- assert indexCnt(ig1, CACHE_NAME) == 1;
+ checkDynamicSchemaChanges(node, SQL_CACHE_NAME);
}
/** */
- private void waitForCheckpoint(final AtomicInteger cnt) throws IgniteInterruptedCheckedException {
- final int i = cnt.get();
+ private int indexCnt(IgniteEx node, String cacheName) {
+ DynamicCacheDescriptor desc = node.context().cache().cacheDescriptor(cacheName);
+
+ int cnt = 0;
- GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return cnt.get() > i;
+ if (desc != null) {
+ QuerySchema schema = desc.schema();
+ if (schema != null) {
+ for (QueryEntity entity : schema.entities())
+ cnt += entity.getIndexes().size();
}
- }, 2000);
+ }
+ return cnt;
}
/** */
- private int indexCnt(IgniteEx node, String cacheName) {
-
+ private int colsCnt(IgniteEx node, String cacheName) {
DynamicCacheDescriptor desc = node.context().cache().cacheDescriptor(cacheName);
int cnt = 0;
- for (QueryEntity entity : desc.schema().entities())
- cnt += entity.getIndexes().size();
+ if (desc != null) {
+ QuerySchema schema = desc.schema();
+ if (schema != null) {
+
+ for (QueryEntity entity : schema.entities())
+ cnt += entity.getFields().size();
+ }
+ }
return cnt;
}
@@ -236,6 +251,50 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest
}
/**
+ * @param node Node whose checkpoint to wait for.
+ * @return Latch released when checkpoint happens.
+ */
+ private CountDownLatch checkpointLatch(IgniteEx node) {
+ final CountDownLatch cnt = new CountDownLatch(1);
+
+ GridCacheDatabaseSharedManager db = (GridCacheDatabaseSharedManager)node.context().cache().context().database();
+
+ db.addCheckpointListener(new DbCheckpointListener() {
+ @Override public void onCheckpointBegin(Context ctx) {
+ cnt.countDown();
+ }
+ });
+
+ return cnt;
+ }
+
+ /**
+ * Create dynamic index and column.
+ * @param node Node.
+ * @param schema Schema name.
+ */
+ private void makeDynamicSchemaChanges(IgniteEx node, String schema) {
+ node.context().query().querySqlFieldsNoCache(
+ new SqlFieldsQuery("create index \"my_idx\" on \"Person\" (\"id\", \"name\")").setSchema(schema), false)
+ .getAll();
+
+ node.context().query().querySqlFieldsNoCache(
+ new SqlFieldsQuery("alter table \"Person\" add column \"age\" int").setSchema(schema), false)
+ .getAll();
+ }
+
+ /**
+ * Check that dynamically created schema objects are in place.
+ * @param node Node.
+ * @param cacheName Cache name.
+ */
+ private void checkDynamicSchemaChanges(IgniteEx node, String cacheName) {
+ assertEquals(1, indexCnt(node, cacheName));
+
+ assertEquals(3, colsCnt(node, cacheName));
+ }
+
+ /**
*
*/
protected static class Person implements Serializable {
@@ -269,21 +328,19 @@ public class IgnitePersistentStoreSchemaLoadTest extends GridCommonAbstractTest
if (o == null || getClass() != o.getClass())
return false;
- IgnitePersistentStoreSchemaLoadTest.Person person = (IgnitePersistentStoreSchemaLoadTest.Person)o;
+ IgnitePersistentStoreSchemaLoadTest.Person person = (IgnitePersistentStoreSchemaLoadTest.Person) o;
- if (id != person.id)
- return false;
+ return id == person.id && (name != null ? name.equals(person.name) : person.name == null);
- return name != null ? name.equals(person.name) : person.name == null;
}
/** {@inheritDoc} */
@Override public int hashCode() {
- int result = id;
+ int res = id;
- result = 31 * result + (name != null ? name.hashCode() : 0);
+ res = 31 * res + (name != null ? name.hashCode() : 0);
- return result;
+ return res;
}
}
}
[13/35] ignite git commit: Updated github docs for .NET and C++
Posted by sb...@apache.org.
Updated github docs for .NET and C++
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f6f6c628
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f6f6c628
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f6f6c628
Branch: refs/heads/ignite-3479
Commit: f6f6c6285f1040265f97e056ae2cb2a62b801057
Parents: 8cc6a59
Author: Denis Magda <dm...@gridgain.com>
Authored: Thu Sep 21 13:24:47 2017 -0700
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu Sep 21 13:24:47 2017 -0700
----------------------------------------------------------------------
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f6f6c628/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 2306d16..d4033e3 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@ With the Ignite Persistence enabled, you no longer need to keep all the data and
<p align="center">
<a href="https://apacheignite.readme.io/docs/distributed-persistent-store">
- <img src="https://ignite.apache.org/images/native_persistence.png" width="400px"/>
+ <img src="https://ignite.apache.org/images/native_persistence.png?renew" width="400px"/>
</a>
</p>
[34/35] ignite git commit: Merge remote-tracking branch
'remotes/origin/master' into ignite-3478
Posted by sb...@apache.org.
Merge remote-tracking branch 'remotes/origin/master' into ignite-3478
# Conflicts:
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteInternalTx.java
# modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxAdapter.java
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/0c4325a3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/0c4325a3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/0c4325a3
Branch: refs/heads/ignite-3479
Commit: 0c4325a3e2f49c44971a6dd6b14d73a15f26a84f
Parents: 3ad260d
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 14:19:17 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 14:19:17 2017 +0300
----------------------------------------------------------------------
.../near/GridNearTxFastFinishFuture.java | 6 ++----
.../near/GridNearTxFinishAndAckFuture.java | 11 +++++++---
.../near/GridNearTxFinishFuture.java | 11 ++++------
.../cache/distributed/near/GridNearTxLocal.java | 22 ++++++++++----------
.../distributed/near/NearTxFinishFuture.java | 3 ++-
5 files changed, 27 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c4325a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
index 7222697..de30090 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFastFinishFuture.java
@@ -51,10 +51,8 @@ public class GridNearTxFastFinishFuture extends GridFutureAdapter<IgniteInternal
return commit;
}
- /**
- *
- */
- public void finish() {
+ /** {@inheritDoc} */
+ public void finish(boolean commit, boolean clearThreadMap) {
try {
if (commit) {
tx.state(PREPARING);
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c4325a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishAndAckFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishAndAckFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishAndAckFuture.java
index b044e6f..8474ab7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishAndAckFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishAndAckFuture.java
@@ -40,9 +40,14 @@ public class GridNearTxFinishAndAckFuture extends GridFutureAdapter<IgniteIntern
}
/** {@inheritDoc} */
- public void finish(boolean commit) {
+ @Override public boolean commit() {
+ return finishFut.commit();
+ }
+
+ /** {@inheritDoc} */
+ public void finish(boolean commit, boolean clearThreadMap) {
if (commit) {
- finishFut.finish(true);
+ finishFut.finish(true, clearThreadMap);
finishFut.listen(new IgniteInClosure<GridNearTxFinishFuture>() {
@Override public void apply(final GridNearTxFinishFuture fut) {
@@ -87,7 +92,7 @@ public class GridNearTxFinishAndAckFuture extends GridFutureAdapter<IgniteIntern
});
}
else {
- finishFut.finish(false);
+ finishFut.finish(false, clearThreadMap);
finishFut.listen(new IgniteInClosure<IgniteInternalFuture>() {
@Override public void apply(IgniteInternalFuture fut) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c4325a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
index 92f6b86..5f18e9b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxFinishFuture.java
@@ -402,15 +402,12 @@ public final class GridNearTxFinishFuture<K, V> extends GridCacheCompoundIdentit
fut.getClass() == CheckRemoteTxMiniFuture.class;
}
- /**
- * Initializes future.
- *
- * @param commit Commit flag.
- * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
- */
- @SuppressWarnings("ForLoopReplaceableByForEach")
/** {@inheritDoc} */
+ @SuppressWarnings("ForLoopReplaceableByForEach")
public void finish(boolean commit, boolean clearThreadMap) {
+ if (!cctx.mvcc().addFuture(this, futureId()))
+ return;
+
if (tx.onNeedCheckBackup()) {
assert tx.onePhaseCommit();
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c4325a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index 91e1c07..188200e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -3199,6 +3199,12 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
commitNearTxLocalAsync().get();
}
+ private NearTxFinishFuture commitFuture() {
+ GridNearTxFinishFuture fut = new GridNearTxFinishFuture<>(cctx, this, true);
+
+ return txState.mvccEnabled(cctx) ? new GridNearTxFinishAndAckFuture(fut) : fut;
+ }
+
/**
* @return Finish future.
*/
@@ -3217,18 +3223,16 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, true)))
return chainFinishFuture(finishFut, true);
- fut0.finish();
+ fut0.finish(true, true);
return fut0;
}
- final GridNearTxFinishFuture fut0;
+ final NearTxFinishFuture fut0;
- if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFinishFuture<>(cctx, this, true)))
+ if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = commitFuture()))
return chainFinishFuture(finishFut, true);
- cctx.mvcc().addFuture(fut0, fut0.futureId());
-
final IgniteInternalFuture<?> prepareFut = prepareNearTxLocal();
prepareFut.listen(new CI1<IgniteInternalFuture<?>>() {
@@ -3299,7 +3303,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (!FINISH_FUT_UPD.compareAndSet(this, null, fut0 = new GridNearTxFastFinishFuture(this, false)))
return chainFinishFuture(finishFut, false);
- fut0.finish();
+ fut0.finish(false, onTimeout);
return fut0;
}
@@ -3376,12 +3380,8 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
assert rollbackFut.isDone() : rollbackFut;
}
- else {
- if (!cctx.mvcc().addFuture(rollbackFut, rollbackFut.futureId()))
- return;
-
+ else
rollbackFut.finish(false, true);
- }
}
}
});
http://git-wip-us.apache.org/repos/asf/ignite/blob/0c4325a3/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
index 9223c9f..19dca86 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/NearTxFinishFuture.java
@@ -31,6 +31,7 @@ public interface NearTxFinishFuture extends IgniteInternalFuture<IgniteInternalT
/**
* @param commit {@code True} to commit, otherwise rollback.
+ * @param clearThreadMap If {@code true} removes {@link GridNearTxLocal} from thread map.
*/
- public void finish(boolean commit);
+ public void finish(boolean commit, boolean clearThreadMap);
}
[10/35] ignite git commit: IGNITE-6294: ODBC: implemented SQLSTATE
management. This closes #2716.
Posted by sb...@apache.org.
IGNITE-6294: ODBC: implemented SQLSTATE management. This closes #2716.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4b4acd39
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4b4acd39
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4b4acd39
Branch: refs/heads/ignite-3479
Commit: 4b4acd39899bc02c6c4836e6848daed83b02dca6
Parents: ea0dcc4
Author: Igor Sapego <ig...@gmail.com>
Authored: Thu Sep 21 20:02:33 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Thu Sep 21 20:02:33 2017 +0300
----------------------------------------------------------------------
.../odbc/odbc/OdbcConnectionContext.java | 6 +-
.../processors/odbc/odbc/OdbcMessageParser.java | 17 +-
.../odbc/odbc/OdbcQueryExecuteBatchResult.java | 16 +-
.../odbc/odbc/OdbcRequestHandler.java | 54 +++-
.../processors/odbc/odbc/OdbcUtils.java | 24 +-
modules/platforms/cpp/odbc-test/Makefile.am | 1 +
.../cpp/odbc-test/project/vs/odbc-test.vcxproj | 1 +
.../project/vs/odbc-test.vcxproj.filters | 3 +
.../platforms/cpp/odbc-test/src/errors_test.cpp | 289 +++++++++++++++++++
.../platforms/cpp/odbc-test/src/parser_test.cpp | 4 +-
.../cpp/odbc-test/src/queries_test.cpp | 16 +-
modules/platforms/cpp/odbc/include/Makefile.am | 1 +
.../cpp/odbc/include/ignite/odbc/common_types.h | 131 ++++++++-
.../ignite/odbc/diagnostic/diagnosable.h | 9 +
.../odbc/diagnostic/diagnosable_adapter.h | 8 +
.../cpp/odbc/include/ignite/odbc/message.h | 48 +--
.../cpp/odbc/include/ignite/odbc/odbc_error.h | 94 ++++++
.../cpp/odbc/include/ignite/odbc/parser.h | 35 ++-
.../platforms/cpp/odbc/project/vs/odbc.vcxproj | 1 +
.../cpp/odbc/project/vs/odbc.vcxproj.filters | 3 +
modules/platforms/cpp/odbc/src/common_types.cpp | 51 ++++
modules/platforms/cpp/odbc/src/connection.cpp | 25 +-
.../odbc/src/diagnostic/diagnosable_adapter.cpp | 8 +
.../odbc/src/diagnostic/diagnostic_record.cpp | 105 ++++++-
modules/platforms/cpp/odbc/src/message.cpp | 37 ++-
.../cpp/odbc/src/query/batch_query.cpp | 13 +-
.../odbc/src/query/column_metadata_query.cpp | 11 +-
.../platforms/cpp/odbc/src/query/data_query.cpp | 31 +-
.../cpp/odbc/src/query/table_metadata_query.cpp | 11 +-
modules/platforms/cpp/odbc/src/statement.cpp | 11 +-
30 files changed, 960 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
index f2450e2..a4af478 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcConnectionContext.java
@@ -32,10 +32,10 @@ import org.apache.ignite.internal.util.GridSpinBusyLock;
*/
public class OdbcConnectionContext implements ClientListenerConnectionContext {
/** Version 2.1.0. */
- private static final ClientListenerProtocolVersion VER_2_1_0 = ClientListenerProtocolVersion.create(2, 1, 0);
+ public static final ClientListenerProtocolVersion VER_2_1_0 = ClientListenerProtocolVersion.create(2, 1, 0);
/** Version 2.1.5: added "lazy" flag. */
- private static final ClientListenerProtocolVersion VER_2_1_5 = ClientListenerProtocolVersion.create(2, 1, 5);
+ public static final ClientListenerProtocolVersion VER_2_1_5 = ClientListenerProtocolVersion.create(2, 1, 5);
/** Current version. */
private static final ClientListenerProtocolVersion CURRENT_VER = VER_2_1_5;
@@ -101,7 +101,7 @@ public class OdbcConnectionContext implements ClientListenerConnectionContext {
handler = new OdbcRequestHandler(ctx, busyLock, maxCursors, distributedJoins,
enforceJoinOrder, replicatedOnly, collocated, lazy);
- parser = new OdbcMessageParser(ctx);
+ parser = new OdbcMessageParser(ctx, ver);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
index 2ab7cee..04e2e25 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcMessageParser.java
@@ -30,6 +30,7 @@ import org.apache.ignite.internal.binary.streams.BinaryHeapOutputStream;
import org.apache.ignite.internal.binary.streams.BinaryInputStream;
import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl;
import org.apache.ignite.internal.processors.odbc.ClientListenerMessageParser;
+import org.apache.ignite.internal.processors.odbc.ClientListenerProtocolVersion;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.apache.ignite.internal.processors.odbc.SqlListenerUtils;
@@ -51,11 +52,16 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
/** Logger. */
private final IgniteLogger log;
+ /** Protocol version */
+ private final ClientListenerProtocolVersion ver;
+
/**
* @param ctx Context.
+ * @param ver Protocol version.
*/
- public OdbcMessageParser(GridKernalContext ctx) {
+ public OdbcMessageParser(GridKernalContext ctx, ClientListenerProtocolVersion ver) {
this.ctx = ctx;
+ this.ver = ver;
log = ctx.log(getClass());
@@ -194,7 +200,11 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
BinaryThreadLocalContext.get().schemaHolder(), null);
// Writing status.
- writer.writeByte((byte) msg.status());
+ if (ver.compareTo(OdbcConnectionContext.VER_2_1_5) < 0) {
+ writer.writeByte((byte) (msg.status() == ClientListenerResponse.STATUS_SUCCESS ?
+ ClientListenerResponse.STATUS_SUCCESS : ClientListenerResponse.STATUS_FAILED));
+ } else
+ writer.writeInt(msg.status());
if (msg.status() != ClientListenerResponse.STATUS_SUCCESS) {
writer.writeString(msg.error());
@@ -234,6 +244,9 @@ public class OdbcMessageParser implements ClientListenerMessageParser {
if (res.errorMessage() != null) {
writer.writeLong(res.errorSetIdx());
writer.writeString(res.errorMessage());
+
+ if (ver.compareTo(OdbcConnectionContext.VER_2_1_5) >= 0)
+ writer.writeInt(res.errorCode());
}
}
else if (res0 instanceof OdbcQueryFetchResult) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
index ee02fd6..c8f61dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcQueryExecuteBatchResult.java
@@ -17,6 +17,7 @@
package org.apache.ignite.internal.processors.odbc.odbc;
+import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
import org.jetbrains.annotations.Nullable;
/**
@@ -29,6 +30,9 @@ public class OdbcQueryExecuteBatchResult {
/** Index of the set which caused an error. */
private final long errorSetIdx;
+ /** Error code. */
+ private final int errorCode;
+
/** Error message. */
private final String errorMessage;
@@ -39,17 +43,20 @@ public class OdbcQueryExecuteBatchResult {
this.rowsAffected = rowsAffected;
this.errorSetIdx = -1;
this.errorMessage = null;
+ this.errorCode = ClientListenerResponse.STATUS_SUCCESS;
}
/**
* @param rowsAffected Number of rows affected by the query.
* @param errorSetIdx Sets processed.
+ * @param errorCode Error code.
* @param errorMessage Error message.
*/
- public OdbcQueryExecuteBatchResult(long rowsAffected, long errorSetIdx, String errorMessage) {
+ public OdbcQueryExecuteBatchResult(long rowsAffected, long errorSetIdx, int errorCode, String errorMessage) {
this.rowsAffected = rowsAffected;
this.errorSetIdx = errorSetIdx;
this.errorMessage = errorMessage;
+ this.errorCode = errorCode;
}
/**
@@ -72,4 +79,11 @@ public class OdbcQueryExecuteBatchResult {
@Nullable public String errorMessage() {
return errorMessage;
}
+
+ /**
+ * @return Error code.
+ */
+ public int errorCode() {
+ return errorCode;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
index fd31f39..07b41f3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcRequestHandler.java
@@ -35,6 +35,7 @@ import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.binary.BinaryWriterExImpl;
import org.apache.ignite.internal.binary.GridBinaryMarshaller;
import org.apache.ignite.internal.processors.cache.QueryCursorImpl;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequest;
import org.apache.ignite.internal.processors.odbc.ClientListenerRequestHandler;
import org.apache.ignite.internal.processors.odbc.ClientListenerResponse;
@@ -42,6 +43,7 @@ import org.apache.ignite.internal.processors.odbc.odbc.escape.OdbcEscapeUtils;
import org.apache.ignite.internal.processors.query.GridQueryFieldMetadata;
import org.apache.ignite.internal.processors.query.GridQueryIndexing;
import org.apache.ignite.internal.processors.query.GridQueryTypeDescriptor;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.U;
@@ -125,7 +127,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
OdbcRequest req = (OdbcRequest)req0;
if (!busyLock.enterBusy())
- return new OdbcResponse(OdbcResponse.STATUS_FAILED,
+ return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN,
"Failed to handle ODBC request because node is stopping: " + req);
try {
@@ -152,7 +154,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
return getParamsMeta((OdbcQueryGetParamsMetaRequest)req);
}
- return new OdbcResponse(OdbcResponse.STATUS_FAILED, "Unsupported ODBC request: " + req);
+ return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Unsupported ODBC request: " + req);
}
finally {
busyLock.leaveBusy();
@@ -161,7 +163,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
/** {@inheritDoc} */
@Override public ClientListenerResponse handleException(Exception e, ClientListenerRequest req) {
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, e.toString());
+ return exceptionToResult(e);
}
/** {@inheritDoc} */
@@ -219,7 +221,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
int cursorCnt = qryCursors.size();
if (maxCursors > 0 && cursorCnt >= maxCursors)
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, "Too many open cursors (either close " +
+ return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN, "Too many open cursors (either close " +
"other open cursors or increase the limit through " +
"ClientConnectorConfiguration.maxOpenCursorsPerConnection) [maximum=" + maxCursors +
", current=" + cursorCnt + ']');
@@ -258,7 +260,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
U.error(log, "Failed to execute SQL query [reqId=" + req.requestId() + ", req=" + req + ']', e);
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, OdbcUtils.retrieveH2ErrorMessage(e));
+ return exceptionToResult(e);
}
}
@@ -308,10 +310,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
catch (Exception e) {
U.error(log, "Failed to execute SQL query [reqId=" + req.requestId() + ", req=" + req + ']', e);
- OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
- OdbcUtils.retrieveH2ErrorMessage(e));
-
- return new OdbcResponse(res);
+ return exceptionToBatchResult(e, rowsAffected, currentSet);
}
}
@@ -364,7 +363,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
IgniteBiTuple<QueryCursor, Iterator> tuple = qryCursors.get(queryId);
if (tuple == null)
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED,
+ return new OdbcResponse(IgniteQueryErrorCode.UNKNOWN,
"Failed to find query with ID: " + queryId);
CloseCursor(tuple, queryId);
@@ -378,7 +377,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
U.error(log, "Failed to close SQL query [reqId=" + req.requestId() + ", req=" + queryId + ']', e);
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, OdbcUtils.retrieveH2ErrorMessage(e));
+ return exceptionToResult(e);
}
}
@@ -427,7 +426,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
catch (Exception e) {
U.error(log, "Failed to fetch SQL query result [reqId=" + req.requestId() + ", req=" + req + ']', e);
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, OdbcUtils.retrieveH2ErrorMessage(e));
+ return exceptionToResult(e);
}
}
@@ -492,7 +491,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
catch (Exception e) {
U.error(log, "Failed to get columns metadata [reqId=" + req.requestId() + ", req=" + req + ']', e);
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, OdbcUtils.retrieveH2ErrorMessage(e));
+ return exceptionToResult(e);
}
}
@@ -540,7 +539,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
catch (Exception e) {
U.error(log, "Failed to get tables metadata [reqId=" + req.requestId() + ", req=" + req + ']', e);
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, OdbcUtils.retrieveH2ErrorMessage(e));
+ return exceptionToResult(e);
}
}
@@ -571,7 +570,7 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
catch (Exception e) {
U.error(log, "Failed to get params metadata [reqId=" + req.requestId() + ", req=" + req + ']', e);
- return new OdbcResponse(ClientListenerResponse.STATUS_FAILED, OdbcUtils.retrieveH2ErrorMessage(e));
+ return exceptionToResult(e);
}
}
@@ -681,4 +680,29 @@ public class OdbcRequestHandler implements ClientListenerRequestHandler {
return str != null && (F.isEmpty(ptrn) ||
str.toUpperCase().matches(ptrn.toUpperCase().replace("%", ".*").replace("_", ".")));
}
+
+ /**
+ * Create {@link OdbcResponse} bearing appropriate Ignite specific result code if possible
+ * from given {@link Exception}.
+ *
+ * @param e Exception to convert.
+ * @return resulting {@link OdbcResponse}.
+ */
+ private OdbcResponse exceptionToBatchResult(Exception e, long rowsAffected, long currentSet) {
+ OdbcQueryExecuteBatchResult res = new OdbcQueryExecuteBatchResult(rowsAffected, currentSet,
+ OdbcUtils.tryRetrieveSqlErrorCode(e), OdbcUtils.tryRetrieveH2ErrorMessage(e));
+
+ return new OdbcResponse(res);
+ }
+
+ /**
+ * Create {@link OdbcResponse} bearing appropriate Ignite specific result code if possible
+ * from given {@link Exception}.
+ *
+ * @param e Exception to convert.
+ * @return resulting {@link OdbcResponse}.
+ */
+ private OdbcResponse exceptionToResult(Exception e) {
+ return new OdbcResponse(OdbcUtils.tryRetrieveSqlErrorCode(e), OdbcUtils.tryRetrieveH2ErrorMessage(e));
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
index 3578145..98fa045 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/odbc/OdbcUtils.java
@@ -18,7 +18,9 @@
package org.apache.ignite.internal.processors.odbc.odbc;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.cache.query.IgniteQueryErrorCode;
import org.apache.ignite.internal.processors.odbc.SqlListenerDataTypes;
+import org.apache.ignite.internal.processors.query.IgniteSQLException;
/**
* Various ODBC utility methods.
@@ -141,11 +143,29 @@ public class OdbcUtils {
}
/**
- * Retrieves error message from exception.
+ * Tries to retrieve SQL error code of the exception. If the exception is not {@link IgniteSQLException} returns
+ * {@link IgniteQueryErrorCode#UNKNOWN}.
+ *
+ * @param err Error to retrieve code from.
+ * @return Error code.
+ */
+ public static int tryRetrieveSqlErrorCode(Throwable err) {
+ int errorCode = IgniteQueryErrorCode.UNKNOWN;
+
+ if (err instanceof IgniteSQLException)
+ errorCode = ((IgniteSQLException) err).statusCode();
+
+ return errorCode;
+ }
+
+ /**
+ * Tries to retrieve H2 engine error message from exception. If the exception is not of type
+ * "org.h2.jdbc.JdbcSQLException" returns original error message.
+ *
* @param err Exception.
* @return Error message.
*/
- public static String retrieveH2ErrorMessage(Throwable err) {
+ public static String tryRetrieveH2ErrorMessage(Throwable err) {
String msg = err.getMessage();
Throwable e = err.getCause();
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc-test/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/Makefile.am b/modules/platforms/cpp/odbc-test/Makefile.am
index 69b62f9..5c335cf 100644
--- a/modules/platforms/cpp/odbc-test/Makefile.am
+++ b/modules/platforms/cpp/odbc-test/Makefile.am
@@ -79,6 +79,7 @@ ignite_odbc_tests_SOURCES = \
src/sql_outer_join_test.cpp \
src/api_robustness_test.cpp \
src/attributes_test.cpp \
+ src/errors_test.cpp \
../odbc/src/cursor.cpp \
../odbc/src/config/connection_info.cpp \
../odbc/src/app/application_data_buffer.cpp \
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
index 3977a24..6ee999e 100644
--- a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
+++ b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj
@@ -169,6 +169,7 @@
<ClCompile Include="..\..\src\configuration_test.cpp" />
<ClCompile Include="..\..\src\connection_info_test.cpp" />
<ClCompile Include="..\..\src\cursor_test.cpp" />
+ <ClCompile Include="..\..\src\errors_test.cpp" />
<ClCompile Include="..\..\src\meta_queries_test.cpp" />
<ClCompile Include="..\..\src\queries_test.cpp" />
<ClCompile Include="..\..\src\parser_test.cpp" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
index 2a0c7b4..abfe303 100644
--- a/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
+++ b/modules/platforms/cpp/odbc-test/project/vs/odbc-test.vcxproj.filters
@@ -127,6 +127,9 @@
<ClCompile Include="..\..\src\attributes_test.cpp">
<Filter>Code</Filter>
</ClCompile>
+ <ClCompile Include="..\..\src\errors_test.cpp">
+ <Filter>Code</Filter>
+ </ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="..\..\include\test_type.h">
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc-test/src/errors_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/src/errors_test.cpp b/modules/platforms/cpp/odbc-test/src/errors_test.cpp
new file mode 100644
index 0000000..bac9fd4
--- /dev/null
+++ b/modules/platforms/cpp/odbc-test/src/errors_test.cpp
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef _WIN32
+# include <windows.h>
+#endif
+
+#include <sql.h>
+#include <sqlext.h>
+
+#include <vector>
+#include <string>
+#include <algorithm>
+
+#ifndef _MSC_VER
+# define BOOST_TEST_DYN_LINK
+#endif
+
+#include <boost/regex.hpp>
+#include <boost/test/unit_test.hpp>
+
+#include "ignite/ignite.h"
+#include "ignite/common/fixed_size_array.h"
+#include "ignite/ignition.h"
+#include "ignite/impl/binary/binary_utils.h"
+#include "ignite/binary/binary_object.h"
+
+#include "test_type.h"
+#include "complex_type.h"
+#include "test_utils.h"
+
+using namespace ignite;
+using namespace ignite::cache;
+using namespace ignite::cache::query;
+using namespace ignite::common;
+using namespace ignite_test;
+using namespace ignite::binary;
+using namespace ignite::impl::binary;
+using namespace ignite::impl::interop;
+
+using namespace boost::unit_test;
+
+using ignite::impl::binary::BinaryUtils;
+
+/**
+ * Test setup fixture.
+ */
+struct ErrorTestSuiteFixture
+{
+ /**
+ * Establish connection to node.
+ *
+ * @param connectStr Connection string.
+ */
+ void Connect(const std::string& connectStr)
+ {
+ // Allocate an environment handle
+ SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &env);
+
+ BOOST_REQUIRE(env != NULL);
+
+ // We want ODBC 3 support
+ SQLSetEnvAttr(env, SQL_ATTR_ODBC_VERSION, reinterpret_cast<void*>(SQL_OV_ODBC3), 0);
+
+ // Allocate a connection handle
+ SQLAllocHandle(SQL_HANDLE_DBC, env, &dbc);
+
+ BOOST_REQUIRE(dbc != NULL);
+
+ // Connect string
+ std::vector<SQLCHAR> connectStr0;
+
+ connectStr0.reserve(connectStr.size() + 1);
+ std::copy(connectStr.begin(), connectStr.end(), std::back_inserter(connectStr0));
+
+ SQLCHAR outstr[ODBC_BUFFER_SIZE];
+ SQLSMALLINT outstrlen;
+
+ // Connecting to ODBC server.
+ SQLRETURN ret = SQLDriverConnect(dbc, NULL, &connectStr0[0], static_cast<SQLSMALLINT>(connectStr0.size()),
+ outstr, sizeof(outstr), &outstrlen, SQL_DRIVER_COMPLETE);
+
+ if (!SQL_SUCCEEDED(ret))
+ {
+ Ignition::StopAll(true);
+
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_DBC, dbc));
+ }
+
+ // Allocate a statement handle
+ SQLAllocHandle(SQL_HANDLE_STMT, dbc, &stmt);
+
+ BOOST_REQUIRE(stmt != NULL);
+ }
+
+ void Disconnect()
+ {
+ // Releasing statement handle.
+ SQLFreeHandle(SQL_HANDLE_STMT, stmt);
+
+ // Disconneting from the server.
+ SQLDisconnect(dbc);
+
+ // Releasing allocated handles.
+ SQLFreeHandle(SQL_HANDLE_DBC, dbc);
+ SQLFreeHandle(SQL_HANDLE_ENV, env);
+ }
+
+ static Ignite StartAdditionalNode(const char* name)
+ {
+#ifdef IGNITE_TESTS_32
+ return StartNode("queries-test-32.xml", name);
+#else
+ return StartNode("queries-test.xml", name);
+#endif
+ }
+
+ /**
+ * Constructor.
+ */
+ ErrorTestSuiteFixture() :
+ env(NULL),
+ dbc(NULL),
+ stmt(NULL)
+ {
+ // No-op.
+ }
+
+ /**
+ * Destructor.
+ */
+ ~ErrorTestSuiteFixture()
+ {
+ Disconnect();
+
+ Ignition::StopAll(true);
+ }
+
+ /** Frist cache instance. */
+ //Cache<int64_t, TestType> cache;
+
+ /** ODBC Environment. */
+ SQLHENV env;
+
+ /** ODBC Connect. */
+ SQLHDBC dbc;
+
+ /** ODBC Statement. */
+ SQLHSTMT stmt;
+};
+
+BOOST_FIXTURE_TEST_SUITE(ErrorTestSuite, ErrorTestSuiteFixture)
+
+BOOST_AUTO_TEST_CASE(TestConnectFail)
+{
+ // Allocate an environment handle
+ SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &env);
+
+ BOOST_REQUIRE(env != NULL);
+
+ // We want ODBC 3 support
+ SQLSetEnvAttr(env, SQL_ATTR_ODBC_VERSION, reinterpret_cast<void*>(SQL_OV_ODBC3), 0);
+
+ // Allocate a connection handle
+ SQLAllocHandle(SQL_HANDLE_DBC, env, &dbc);
+
+ BOOST_REQUIRE(dbc != NULL);
+
+ // Connect string
+ SQLCHAR connectStr[] = "DRIVER={Apache Ignite};ADDRESS=127.0.0.1:9999;SCHEMA=cache";
+
+ SQLCHAR outstr[ODBC_BUFFER_SIZE];
+ SQLSMALLINT outstrlen;
+
+ // Connecting to ODBC server.
+ SQLRETURN ret = SQLDriverConnect(dbc, NULL, connectStr, SQL_NTS,
+ outstr, sizeof(outstr), &outstrlen, SQL_DRIVER_COMPLETE);
+
+ BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
+ BOOST_CHECK_EQUAL(GetOdbcErrorState(SQL_HANDLE_DBC, dbc), "08001");
+}
+
+BOOST_AUTO_TEST_CASE(TestDuplicateKey)
+{
+ StartAdditionalNode("Node1");
+
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+ SQLCHAR insertReq[] = "INSERT INTO TestType(_key, strField) VALUES(1, 'some')";
+
+ SQLRETURN ret;
+
+ ret = SQLExecDirect(stmt, insertReq, SQL_NTS);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ ret = SQLExecDirect(stmt, insertReq, SQL_NTS);
+
+ BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
+ BOOST_CHECK_EQUAL(GetOdbcErrorState(SQL_HANDLE_STMT, stmt), "23000");
+}
+
+BOOST_AUTO_TEST_CASE(TestUpdateKey)
+{
+ StartAdditionalNode("Node1");
+
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache");
+
+ SQLCHAR insertReq[] = "INSERT INTO TestType(_key, strField) VALUES(1, 'some')";
+
+ SQLRETURN ret;
+
+ ret = SQLExecDirect(stmt, insertReq, SQL_NTS);
+
+ if (!SQL_SUCCEEDED(ret))
+ BOOST_FAIL(GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt));
+
+ SQLCHAR updateReq[] = "UPDATE TestType SET _key=2 WHERE _key=1";
+
+ ret = SQLExecDirect(stmt, updateReq, SQL_NTS);
+
+ BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
+ BOOST_CHECK_EQUAL(GetOdbcErrorState(SQL_HANDLE_STMT, stmt), "42000");
+}
+
+BOOST_AUTO_TEST_CASE(TestTableNotFound)
+{
+ StartAdditionalNode("Node1");
+
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=PUBLIC");
+
+ SQLCHAR req[] = "DROP TABLE Nonexisting";
+
+ SQLRETURN ret;
+
+ ret = SQLExecDirect(stmt, req, SQL_NTS);
+
+ BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
+ BOOST_CHECK_EQUAL(GetOdbcErrorState(SQL_HANDLE_STMT, stmt), "42S02");
+}
+
+BOOST_AUTO_TEST_CASE(TestIndexNotFound)
+{
+ StartAdditionalNode("Node1");
+
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=PUBLIC");
+
+ SQLCHAR req[] = "DROP INDEX Nonexisting";
+
+ SQLRETURN ret;
+
+ ret = SQLExecDirect(stmt, req, SQL_NTS);
+
+ BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
+ BOOST_CHECK_EQUAL(GetOdbcErrorState(SQL_HANDLE_STMT, stmt), "42S12");
+}
+
+BOOST_AUTO_TEST_CASE(TestSyntaxError)
+{
+ StartAdditionalNode("Node1");
+
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=PUBLIC");
+
+ SQLCHAR req[] = "INSERT INTO TestType(_key, fasf) VALUES(1, 'some')";
+
+ SQLRETURN ret;
+
+ ret = SQLExecDirect(stmt, req, SQL_NTS);
+
+ BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
+ BOOST_CHECK_EQUAL(GetOdbcErrorState(SQL_HANDLE_STMT, stmt), "42000");
+}
+
+BOOST_AUTO_TEST_SUITE_END()
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc-test/src/parser_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/src/parser_test.cpp b/modules/platforms/cpp/odbc-test/src/parser_test.cpp
index 7c8a73d..51539ae 100644
--- a/modules/platforms/cpp/odbc-test/src/parser_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/parser_test.cpp
@@ -27,7 +27,7 @@ using namespace ignite::odbc;
struct TestMessage
{
- TestMessage()
+ TestMessage() : a(0), b()
{
// No-op.
}
@@ -48,7 +48,7 @@ struct TestMessage
writer.WriteString(b.data(), static_cast<int32_t>(b.size()));
}
- void Read(ignite::impl::binary::BinaryReaderImpl& reader)
+ void Read(ignite::impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
a = reader.ReadInt32();
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc-test/src/queries_test.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc-test/src/queries_test.cpp b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
index e7168fe..4c7e402 100644
--- a/modules/platforms/cpp/odbc-test/src/queries_test.cpp
+++ b/modules/platforms/cpp/odbc-test/src/queries_test.cpp
@@ -734,11 +734,25 @@ BOOST_FIXTURE_TEST_SUITE(QueriesTestSuite, QueriesTestSuiteFixture)
BOOST_AUTO_TEST_CASE(TestLegacyConnection)
{
Connect("DRIVER={Apache Ignite};SERVER=127.0.0.1;PORT=11110;SCHEMA=cache");
+
+ InsertTestStrings(10, false);
+ InsertTestBatch(11, 20, 9);
}
BOOST_AUTO_TEST_CASE(TestConnectionProtocolVersion_2_1_0)
{
Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache;PROTOCOL_VERSION=2.1.0");
+
+ InsertTestStrings(10, false);
+ InsertTestBatch(11, 20, 9);
+}
+
+BOOST_AUTO_TEST_CASE(TestConnectionProtocolVersion_2_1_5)
+{
+ Connect("DRIVER={Apache Ignite};ADDRESS=127.0.0.1:11110;SCHEMA=cache;PROTOCOL_VERSION=2.1.5");
+
+ InsertTestStrings(10, false);
+ InsertTestBatch(11, 20, 9);
}
BOOST_AUTO_TEST_CASE(TestTwoRowsInt8)
@@ -2119,7 +2133,7 @@ BOOST_AUTO_TEST_CASE(TestErrorMessage)
BOOST_REQUIRE_EQUAL(ret, SQL_ERROR);
std::string error = GetOdbcErrorMessage(SQL_HANDLE_STMT, stmt);
- std::string pattern = "HY000: Table \"B\" not found; SQL statement:\\vSELECT a FROM B.*";
+ std::string pattern = "42000: Table \"B\" not found; SQL statement:\\vSELECT a FROM B.*";
boost::cmatch what;
if (!boost::regex_match(error.c_str(), what, boost::regex(pattern)))
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/Makefile.am
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/Makefile.am b/modules/platforms/cpp/odbc/include/Makefile.am
index 92c85c5..7c64754 100644
--- a/modules/platforms/cpp/odbc/include/Makefile.am
+++ b/modules/platforms/cpp/odbc/include/Makefile.am
@@ -51,6 +51,7 @@ noinst_HEADERS = \
ignite/odbc/diagnostic/diagnosable.h \
ignite/odbc/diagnostic/diagnosable_adapter.h \
ignite/odbc/connection.h \
+ ignite/odbc/odbc_error.h \
ignite/odbc/message.h \
ignite/odbc/cursor.h \
ignite/odbc/common_types.h \
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
index 0a3b02f..349147f 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/common_types.h
@@ -77,12 +77,42 @@ namespace ignite
*/
S01S02_OPTION_VALUE_CHANGED,
+ /** Restricted data type attribute violation. */
+ S07006_RESTRICTION_VIOLATION,
+
/** String data, length mismatch. */
S22026_DATA_LENGTH_MISMATCH,
+ /** Integrity constraint violation. */
+ S23000_INTEGRITY_CONSTRAINT_VIOLATION,
+
/** Invalid cursor state. */
S24000_INVALID_CURSOR_STATE,
+ /** Invalid schema name. */
+ S3F000_INVALID_SCHEMA_NAME,
+
+ /** Syntax error or access violation. */
+ S42000_SYNTAX_ERROR_OR_ACCESS_VIOLATION,
+
+ /** Base table or view already exists. */
+ S42S01_TABLE_OR_VIEW_ALREADY_EXISTS,
+
+ /** Base table or view not found. */
+ S42S02_TABLE_OR_VIEW_NOT_FOUND,
+
+ /** Index already exists. */
+ S42S11_INDEX_ALREADY_EXISTS,
+
+ /** Index not found. */
+ S42S12_INDEX_NOT_FOUND,
+
+ /** Column already exists. */
+ S42S21_COLUMN_ALREADY_EXISTS,
+
+ /** Column not found. */
+ S42S22_COLUMN_NOT_FOUND,
+
/** Invalid descriptor index. */
S07009_INVALID_DESCRIPTOR_INDEX,
@@ -102,6 +132,12 @@ namespace ignite
/** The connection specified was not open. */
S08003_NOT_CONNECTED,
+ /** Server rejected the connection. */
+ S08004_CONNECTION_REJECTED,
+
+ /** Communication link failure. */
+ S08S01_LINK_FAILURE,
+
/**
* An error occurred for which there was no specific SQLSTATE
* and for which no implementation-specific SQLSTATE was defined.
@@ -120,6 +156,9 @@ namespace ignite
*/
SHY003_INVALID_APPLICATION_BUFFER_TYPE,
+ /** Invalid SQL data type. */
+ SHY004_INVALID_SQL_DATA_TYPE,
+
/** Invalid use of null pointer. */
SHY009_INVALID_USE_OF_NULL_POINTER,
@@ -155,7 +194,7 @@ namespace ignite
* The connection timeout period expired before the data source
* responded to the request.
*/
- SHYT01_CONNECTIOIN_TIMEOUT,
+ SHYT01_CONNECTION_TIMEOUT,
/**
* Driver does not support this function.
@@ -245,6 +284,88 @@ namespace ignite
};
};
+ struct ResponseStatus
+ {
+ enum Type
+ {
+ /** Operation completed successfully. */
+ SUCCESS = 0,
+
+ /* 1xxx - parsing errors */
+
+ /** Unknown error, or the one without specific code. */
+ UNKNOWN_ERROR = 1,
+
+ /** General parsing error - for the cases when there's no more specific code available. */
+ PARSING_FAILURE = 1001,
+
+ /** Requested operation is not supported. */
+ UNSUPPORTED_OPERATION = 1002,
+
+ /* 2xxx - analysis errors */
+
+ /** Code encountered SQL statement of some type that it did not expect in current analysis context. */
+ UNEXPECTED_OPERATION = 2001,
+
+ /** Code encountered SQL expression of some type that it did not expect in current analysis context. */
+ UNEXPECTED_ELEMENT_TYPE = 2002,
+
+ /** Analysis detected that the statement is trying to directly UPDATE key or its fields. */
+ KEY_UPDATE = 2003,
+
+ /* 3xxx - database API related runtime errors */
+ /** Required table not found. */
+ TABLE_NOT_FOUND = 3001,
+
+ /** Required table does not have a descriptor set. */
+ NULL_TABLE_DESCRIPTOR = 3002,
+
+ /** Statement type does not match that declared by JDBC driver. */
+ STMT_TYPE_MISMATCH = 3003,
+
+ /** DROP TABLE failed. */
+ TABLE_DROP_FAILED = 3004,
+
+ /** Index already exists. */
+ INDEX_ALREADY_EXISTS = 3005,
+
+ /** Index does not exist. */
+ INDEX_NOT_FOUND = 3006,
+
+ /** Required table already exists. */
+ TABLE_ALREADY_EXISTS = 3007,
+
+ /** Required column not found. */
+ COLUMN_NOT_FOUND = 3008,
+
+ /** Required column already exists. */
+ COLUMN_ALREADY_EXISTS = 3009,
+
+ /** Conversion failure. */
+ CONVERSION_FAILED = 3013,
+
+ /* 4xxx - cache related runtime errors */
+
+ /** Attempt to INSERT a key that is already in cache. */
+ DUPLICATE_KEY = 4001,
+
+ /** Attempt to UPDATE or DELETE a key whose value has been updated concurrently by someone else. */
+ CONCURRENT_UPDATE = 4002,
+
+ /** Attempt to INSERT or MERGE {@code null} key. */
+ NULL_KEY = 4003,
+
+ /** Attempt to INSERT or MERGE {@code null} value. */
+ NULL_VALUE = 4004,
+
+ /** EntryProcessor has thrown an exception during IgniteCache::invokeAll. */
+ ENTRY_PROCESSING = 4005,
+
+ /** Cache not found. */
+ CACHE_NOT_FOUND = 4006
+ };
+ };
+
/**
* Convert internal Ignite type into ODBC SQL return code.
*
@@ -269,7 +390,13 @@ namespace ignite
*/
EnvironmentAttribute::Type EnvironmentAttributeToInternal(int32_t attr);
-
+ /**
+ * Convert request response status to SQL state.
+ *
+ * @param status Response status.
+ * @return SQL state.
+ */
+ SqlState::Type ResponseStatusToSqlState(int32_t status);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
index 2ec1b5b..46d79ce 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable.h
@@ -24,6 +24,8 @@ namespace ignite
{
namespace odbc
{
+ class OdbcError;
+
namespace diagnostic
{
/**
@@ -73,6 +75,13 @@ namespace ignite
*/
virtual void AddStatusRecord(SqlState::Type sqlState, const std::string& message) = 0;
+ /**
+ * Add new status record.
+ *
+ * @param err Error.
+ */
+ virtual void AddStatusRecord(const OdbcError& err) = 0;
+
protected:
/**
* Default constructor.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable_adapter.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable_adapter.h b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable_adapter.h
index 45f47f0..461bfc9 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable_adapter.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/diagnostic/diagnosable_adapter.h
@@ -33,6 +33,7 @@ namespace ignite
{
namespace odbc
{
+ class OdbcError;
class Connection;
namespace diagnostic
@@ -90,6 +91,13 @@ namespace ignite
*/
virtual void AddStatusRecord(SqlState::Type sqlState, const std::string& message);
+ /**
+ * Add new status record.
+ *
+ * @param err Error.
+ */
+ virtual void AddStatusRecord(const OdbcError& err);
+
protected:
/**
* Constructor.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
index fd7897f..91a808c 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/message.h
@@ -64,16 +64,6 @@ namespace ignite
};
};
- struct ResponseStatus
- {
- enum Type
- {
- SUCCESS = 0,
-
- FAILED = 1
- };
- };
-
/**
* Handshake request.
*/
@@ -411,14 +401,15 @@ namespace ignite
/**
* Read response using provided reader.
* @param reader Reader.
+ * @param ver Protocol version.
*/
- void Read(impl::binary::BinaryReaderImpl& reader);
+ void Read(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver);
/**
* Get request processing status.
* @return Status.
*/
- int8_t GetStatus() const
+ int32_t GetStatus() const
{
return status;
}
@@ -436,11 +427,11 @@ namespace ignite
/**
* Read data if response status is ResponseStatus::SUCCESS.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl&);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl&, const ProtocolVersion&);
private:
/** Request processing status. */
- int8_t status;
+ int32_t status;
/** Error message. */
std::string error;
@@ -493,7 +484,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- void Read(impl::binary::BinaryReaderImpl& reader);
+ void Read(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
private:
/** Handshake accepted. */
@@ -536,7 +527,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
/** Query ID. */
int64_t queryId;
@@ -590,7 +581,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
/** Query ID. */
int64_t queryId;
@@ -645,12 +636,22 @@ namespace ignite
return errorMessage;
}
+ /**
+ * Get error code.
+ * @return Error code.
+ */
+ int32_t GetErrorCode() const
+ {
+ return errorCode;
+ }
+
private:
/**
* Read response using provided reader.
* @param reader Reader.
+ * @param ver Protocol version.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver);
/** Affected rows. */
int64_t affectedRows;
@@ -660,6 +661,9 @@ namespace ignite
/** Error message. */
std::string errorMessage;
+
+ /** Error code. */
+ int32_t errorCode;
};
/**
@@ -693,7 +697,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
/** Query ID. */
int64_t queryId;
@@ -732,7 +736,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
/** Columns metadata. */
meta::ColumnMetaVector meta;
@@ -768,7 +772,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
/** Columns metadata. */
meta::TableMetaVector meta;
@@ -804,7 +808,7 @@ namespace ignite
* Read response using provided reader.
* @param reader Reader.
*/
- virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader);
+ virtual void ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&);
/** Columns metadata. */
std::vector<int8_t> typeIds;
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/ignite/odbc/odbc_error.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/odbc_error.h b/modules/platforms/cpp/odbc/include/ignite/odbc/odbc_error.h
new file mode 100644
index 0000000..361b2b5
--- /dev/null
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/odbc_error.h
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _IGNITE_ODBC_ODBC_ERROR
+#define _IGNITE_ODBC_ODBC_ERROR
+
+#include <string>
+
+#include "ignite/odbc/common_types.h"
+
+namespace ignite
+{
+ namespace odbc
+ {
+ /**
+ * ODBC error.
+ */
+ class OdbcError
+ {
+ public:
+ /**
+ * Constructor.
+ *
+ * @param status SQL status.
+ * @param message Error message.
+ */
+ OdbcError(SqlState::Type status, const std::string& message) :
+ status(status),
+ errMessage(message)
+ {
+ // No-op.
+ }
+
+ /**
+ * Default constructor.
+ */
+ OdbcError() :
+ status(SqlState::UNKNOWN),
+ errMessage()
+ {
+ // No-op.
+ }
+
+ /**
+ * Destructor.
+ */
+ ~OdbcError()
+ {
+ // No-op.
+ }
+
+ /**
+ * Get status.
+ * @return Status.
+ */
+ SqlState::Type GetStatus() const
+ {
+ return status;
+ }
+
+ /**
+ * Get error message.
+ * @return Error message.
+ */
+ const std::string& GetErrorMessage() const
+ {
+ return errMessage;
+ }
+
+ private:
+ /** Status. */
+ SqlState::Type status;
+
+ /** Error message. */
+ std::string errMessage;
+ };
+ }
+}
+
+#endif //_IGNITE_ODBC_ODBC_ERROR
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/include/ignite/odbc/parser.h
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/include/ignite/odbc/parser.h b/modules/platforms/cpp/odbc/include/ignite/odbc/parser.h
index a91af22..bfea67c 100644
--- a/modules/platforms/cpp/odbc/include/ignite/odbc/parser.h
+++ b/modules/platforms/cpp/odbc/include/ignite/odbc/parser.h
@@ -28,6 +28,7 @@
#include <ignite/impl/binary/binary_reader_impl.h>
#include "ignite/odbc/utility.h"
+#include "ignite/odbc/protocol_version.h"
namespace ignite
{
@@ -44,8 +45,13 @@ namespace ignite
/**
* Constructor.
+ * @param cap Initial capasity.
*/
- Parser(int32_t cap = DEFAULT_MEM_ALLOCATION) : inMem(cap), outMem(cap), outStream(&outMem)
+ Parser(int32_t cap = DEFAULT_MEM_ALLOCATION) :
+ protocolVer(ProtocolVersion::GetCurrent()),
+ inMem(cap),
+ outMem(cap),
+ outStream(&outMem)
{
//No-op.
}
@@ -67,7 +73,7 @@ namespace ignite
template<typename MsgT>
void Encode(const MsgT& msg, std::vector<int8_t>& buf)
{
- using namespace ignite::impl::binary;
+ using namespace impl::binary;
ResetState();
@@ -90,7 +96,7 @@ namespace ignite
template<typename MsgT>
void Decode(MsgT& msg, const std::vector<int8_t>& buf)
{
- using namespace ignite::impl::binary;
+ using namespace impl::binary;
if (inMem.Capacity() < static_cast<int32_t>(buf.size()))
inMem.Reallocate(static_cast<int32_t>(buf.size()));
@@ -99,11 +105,21 @@ namespace ignite
inMem.Length(static_cast<int32_t>(buf.size()));
- ignite::impl::interop::InteropInputStream inStream(&inMem);
+ impl::interop::InteropInputStream inStream(&inMem);
BinaryReaderImpl reader(&inStream);
- msg.Read(reader);
+ msg.Read(reader, protocolVer);
+ }
+
+ /**
+ * Set protocol version.
+ *
+ * @param ver Version to set.
+ */
+ void SetProtocolVersion(const ProtocolVersion& ver)
+ {
+ protocolVer = ver;
}
private:
@@ -119,14 +135,17 @@ namespace ignite
outStream.Position(0);
}
+ /** Protocol version. */
+ ProtocolVersion protocolVer;
+
/** Input operational memory. */
- ignite::impl::interop::InteropUnpooledMemory inMem;
+ impl::interop::InteropUnpooledMemory inMem;
/** Output operational memory. */
- ignite::impl::interop::InteropUnpooledMemory outMem;
+ impl::interop::InteropUnpooledMemory outMem;
/** Output stream. */
- ignite::impl::interop::InteropOutputStream outStream;
+ impl::interop::InteropOutputStream outStream;
};
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
index e47f618..c5783ff 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj
@@ -217,6 +217,7 @@
<ClInclude Include="..\..\include\ignite\odbc\meta\column_meta.h" />
<ClInclude Include="..\..\include\ignite\odbc\meta\primary_key_meta.h" />
<ClInclude Include="..\..\include\ignite\odbc\meta\table_meta.h" />
+ <ClInclude Include="..\..\include\ignite\odbc\odbc_error.h" />
<ClInclude Include="..\..\include\ignite\odbc\parser.h" />
<ClInclude Include="..\..\include\ignite\odbc\protocol_version.h" />
<ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h" />
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
index e252d5d..8934625 100644
--- a/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
+++ b/modules/platforms/cpp/odbc/project/vs/odbc.vcxproj.filters
@@ -281,5 +281,8 @@
<ClInclude Include="..\..\include\ignite\odbc\query\batch_query.h">
<Filter>Code\query</Filter>
</ClInclude>
+ <ClInclude Include="..\..\include\ignite\odbc\odbc_error.h">
+ <Filter>Code</Filter>
+ </ClInclude>
</ItemGroup>
</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/common_types.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/common_types.cpp b/modules/platforms/cpp/odbc/src/common_types.cpp
index 09ea3dc..10c4f77 100644
--- a/modules/platforms/cpp/odbc/src/common_types.cpp
+++ b/modules/platforms/cpp/odbc/src/common_types.cpp
@@ -118,6 +118,57 @@ namespace ignite
return EnvironmentAttribute::UNKNOWN;
}
+
+ SqlState::Type ResponseStatusToSqlState(int32_t status)
+ {
+ switch (status)
+ {
+ case ResponseStatus::PARSING_FAILURE:
+ case ResponseStatus::KEY_UPDATE:
+ case ResponseStatus::UNEXPECTED_OPERATION:
+ return SqlState::S42000_SYNTAX_ERROR_OR_ACCESS_VIOLATION;
+
+ case ResponseStatus::UNSUPPORTED_OPERATION:
+ return SqlState::SHYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED;
+
+ case ResponseStatus::UNEXPECTED_ELEMENT_TYPE:
+ return SqlState::SHY004_INVALID_SQL_DATA_TYPE;
+
+ case ResponseStatus::DUPLICATE_KEY:
+ case ResponseStatus::NULL_KEY:
+ case ResponseStatus::NULL_VALUE:
+ return SqlState::S23000_INTEGRITY_CONSTRAINT_VIOLATION;
+
+ case ResponseStatus::TABLE_NOT_FOUND:
+ return SqlState::S42S02_TABLE_OR_VIEW_NOT_FOUND;
+
+ case ResponseStatus::INDEX_ALREADY_EXISTS:
+ return SqlState::S42S11_INDEX_ALREADY_EXISTS;
+
+ case ResponseStatus::INDEX_NOT_FOUND:
+ return SqlState::S42S12_INDEX_NOT_FOUND;
+
+ case ResponseStatus::TABLE_ALREADY_EXISTS:
+ return SqlState::S42S01_TABLE_OR_VIEW_ALREADY_EXISTS;
+
+ case ResponseStatus::COLUMN_NOT_FOUND:
+ return SqlState::S42S22_COLUMN_NOT_FOUND;
+
+ case ResponseStatus::COLUMN_ALREADY_EXISTS:
+ return SqlState::S42S21_COLUMN_ALREADY_EXISTS;
+
+ case ResponseStatus::CACHE_NOT_FOUND:
+ case ResponseStatus::NULL_TABLE_DESCRIPTOR:
+ case ResponseStatus::CONVERSION_FAILED:
+ case ResponseStatus::CONCURRENT_UPDATE:
+ case ResponseStatus::ENTRY_PROCESSING:
+ case ResponseStatus::TABLE_DROP_FAILED:
+ case ResponseStatus::STMT_TYPE_MISMATCH:
+ case ResponseStatus::UNKNOWN_ERROR:
+ default:
+ return SqlState::SHY000_GENERAL_ERROR;
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/connection.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/connection.cpp b/modules/platforms/cpp/odbc/src/connection.cpp
index 777973a..161e1c4 100644
--- a/modules/platforms/cpp/odbc/src/connection.cpp
+++ b/modules/platforms/cpp/odbc/src/connection.cpp
@@ -27,6 +27,7 @@
#include "ignite/odbc/connection.h"
#include "ignite/odbc/message.h"
#include "ignite/odbc/config/configuration.h"
+#include "ignite/odbc/odbc_error.h"
namespace
{
@@ -138,6 +139,8 @@ namespace ignite
if (res == SqlResult::AI_ERROR)
Close();
+ else
+ parser.SetProtocolVersion(config.GetProtocolVersion());
return res;
}
@@ -194,7 +197,7 @@ namespace ignite
void Connection::Send(const int8_t* data, size_t len)
{
if (!connected)
- IGNITE_ERROR_1(IgniteError::IGNITE_ERR_ILLEGAL_STATE, "Connection is not established");
+ throw OdbcError(SqlState::S08003_NOT_CONNECTED, "Connection is not established");
int32_t newLen = static_cast<int32_t>(len + sizeof(OdbcProtocolHeader));
@@ -209,7 +212,7 @@ namespace ignite
size_t sent = SendAll(msg.GetData(), msg.GetSize());
if (sent != len + sizeof(OdbcProtocolHeader))
- IGNITE_ERROR_1(IgniteError::IGNITE_ERR_GENERIC, "Can not send message");
+ throw OdbcError(SqlState::S08S01_LINK_FAILURE, "Can not send message due to connection failure");
LOG_MSG("message sent: (" << msg.GetSize() << " bytes)" << utility::HexDump(msg.GetData(), msg.GetSize()));
}
@@ -240,7 +243,7 @@ namespace ignite
void Connection::Receive(std::vector<int8_t>& msg)
{
if (!connected)
- IGNITE_ERROR_1(IgniteError::IGNITE_ERR_ILLEGAL_STATE, "Connection is not established");
+ throw OdbcError(SqlState::S08003_NOT_CONNECTED, "Connection is not established");
msg.clear();
@@ -249,13 +252,13 @@ namespace ignite
size_t received = ReceiveAll(reinterpret_cast<int8_t*>(&hdr), sizeof(hdr));
if (received != sizeof(hdr))
- IGNITE_ERROR_1(IgniteError::IGNITE_ERR_GENERIC, "Can not receive message header");
+ throw OdbcError(SqlState::S08S01_LINK_FAILURE, "Can not receive message header");
if (hdr.len < 0)
{
Close();
- IGNITE_ERROR_1(IgniteError::IGNITE_ERR_GENERIC, "Message length is negative");
+ throw OdbcError(SqlState::S08S01_LINK_FAILURE, "Protocol error: Message length is negative");
}
if (hdr.len == 0)
@@ -269,7 +272,7 @@ namespace ignite
{
msg.resize(received);
- IGNITE_ERROR_1(IgniteError::IGNITE_ERR_GENERIC, "Can not receive message body");
+ throw OdbcError(SqlState::S08S01_LINK_FAILURE, "Can not receive message body");
}
LOG_MSG("Message received: " << utility::HexDump(&msg[0], msg.size()));
@@ -447,9 +450,15 @@ namespace ignite
{
SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ AddStatusRecord(SqlState::S08004_CONNECTION_REJECTED, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -469,7 +478,7 @@ namespace ignite
<< "driver protocol version introduced in version: "
<< config.GetProtocolVersion().ToString() << ".";
- AddStatusRecord(SqlState::S08001_CANNOT_CONNECT, constructor.str());
+ AddStatusRecord(SqlState::S08004_CONNECTION_REJECTED, constructor.str());
return SqlResult::AI_ERROR;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/diagnostic/diagnosable_adapter.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/diagnostic/diagnosable_adapter.cpp b/modules/platforms/cpp/odbc/src/diagnostic/diagnosable_adapter.cpp
index 6c07c29..096593f 100644
--- a/modules/platforms/cpp/odbc/src/diagnostic/diagnosable_adapter.cpp
+++ b/modules/platforms/cpp/odbc/src/diagnostic/diagnosable_adapter.cpp
@@ -17,6 +17,7 @@
#include "ignite/odbc/log.h"
#include "ignite/odbc/connection.h"
+#include "ignite/odbc/odbc_error.h"
#include "ignite/odbc/diagnostic/diagnosable_adapter.h"
namespace ignite
@@ -46,6 +47,13 @@ namespace ignite
AddStatusRecord(sqlState, message, 0, 0);
}
+
+ void DiagnosableAdapter::AddStatusRecord(const OdbcError& err)
+ {
+ LOG_MSG("Adding new record: " << err.GetErrorMessage());
+
+ AddStatusRecord(err.GetStatus(), err.GetErrorMessage(), 0, 0);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp b/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
index c8e0e81..0a02310 100644
--- a/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
+++ b/modules/platforms/cpp/odbc/src/diagnostic/diagnostic_record.cpp
@@ -40,15 +40,15 @@ namespace
/** SQL state 01S01 constant. */
const std::string STATE_01S01 = "01S01";
- /** SQL state 22026 constant. */
- const std::string STATE_22026 = "22026";
-
- /** SQL state 24000 constant. */
- const std::string STATE_24000 = "24000";
+ /** SQL state 01S02 constant. */
+ const std::string STATE_01S02 = "01S02";
/** SQL state 07009 constant. */
const std::string STATE_07009 = "07009";
+ /** SQL state 07006 constant. */
+ const std::string STATE_07006 = "07006";
+
/** SQL state 08001 constant. */
const std::string STATE_08001 = "08001";
@@ -58,6 +58,45 @@ namespace
/** SQL state 08003 constant. */
const std::string STATE_08003 = "08003";
+ /** SQL state 08004 constant. */
+ const std::string STATE_08004 = "08004";
+
+ /** SQL state 08S01 constant. */
+ const std::string STATE_08S01 = "08S01";
+
+ /** SQL state 22026 constant. */
+ const std::string STATE_22026 = "22026";
+
+ /** SQL state 23000 constant. */
+ const std::string STATE_23000 = "23000";
+
+ /** SQL state 24000 constant. */
+ const std::string STATE_24000 = "24000";
+
+ /** SQL state 3F000 constant. */
+ const std::string STATE_3F000 = "3F000";
+
+ /** SQL state 42000 constant. */
+ const std::string STATE_42000 = "42000";
+
+ /** SQL state 42S01 constant. */
+ const std::string STATE_42S01 = "42S01";
+
+ /** SQL state 42S02 constant. */
+ const std::string STATE_42S02 = "42S02";
+
+ /** SQL state 42S11 constant. */
+ const std::string STATE_42S11 = "42S11";
+
+ /** SQL state 42S12 constant. */
+ const std::string STATE_42S12 = "42S12";
+
+ /** SQL state 42S21 constant. */
+ const std::string STATE_42S21 = "42S21";
+
+ /** SQL state 42S22 constant. */
+ const std::string STATE_42S22 = "42S22";
+
/** SQL state HY000 constant. */
const std::string STATE_HY000 = "HY000";
@@ -67,15 +106,24 @@ namespace
/** SQL state HY003 constant. */
const std::string STATE_HY003 = "HY003";
+ /** SQL state HY004 constant. */
+ const std::string STATE_HY004 = "HY004";
+
/** SQL state HY009 constant. */
const std::string STATE_HY009 = "HY009";
/** SQL state HY010 constant. */
const std::string STATE_HY010 = "HY010";
+ /** SQL state HY090 constant. */
+ const std::string STATE_HY090 = "HY090";
+
/** SQL state HY092 constant. */
const std::string STATE_HY092 = "HY092";
+ /** SQL state HY097 constant. */
+ const std::string STATE_HY097 = "HY097";
+
/** SQL state HY105 constant. */
const std::string STATE_HY105 = "HY105";
@@ -88,9 +136,6 @@ namespace
/** SQL state HYT01 constant. */
const std::string STATE_HYT01 = "HYT01";
- /** SQL state HY090 constant. */
- const std::string STATE_HY090 = "HY090";
-
/** SQL state IM001 constant. */
const std::string STATE_IM001 = "IM001";
}
@@ -230,12 +275,45 @@ namespace ignite
case SqlState::S01S01_ERROR_IN_ROW:
return STATE_01S01;
+ case SqlState::S01S02_OPTION_VALUE_CHANGED:
+ return STATE_01S02;
+
+ case SqlState::S07006_RESTRICTION_VIOLATION:
+ return STATE_07006;
+
case SqlState::S22026_DATA_LENGTH_MISMATCH:
return STATE_22026;
+ case SqlState::S23000_INTEGRITY_CONSTRAINT_VIOLATION:
+ return STATE_23000;
+
case SqlState::S24000_INVALID_CURSOR_STATE:
return STATE_24000;
+ case SqlState::S3F000_INVALID_SCHEMA_NAME:
+ return STATE_3F000;
+
+ case SqlState::S42000_SYNTAX_ERROR_OR_ACCESS_VIOLATION:
+ return STATE_42000;
+
+ case SqlState::S42S01_TABLE_OR_VIEW_ALREADY_EXISTS:
+ return STATE_42S01;
+
+ case SqlState::S42S02_TABLE_OR_VIEW_NOT_FOUND:
+ return STATE_42S02;
+
+ case SqlState::S42S11_INDEX_ALREADY_EXISTS:
+ return STATE_42S11;
+
+ case SqlState::S42S12_INDEX_NOT_FOUND:
+ return STATE_42S12;
+
+ case SqlState::S42S21_COLUMN_ALREADY_EXISTS:
+ return STATE_42S21;
+
+ case SqlState::S42S22_COLUMN_NOT_FOUND:
+ return STATE_42S22;
+
case SqlState::S07009_INVALID_DESCRIPTOR_INDEX:
return STATE_07009;
@@ -248,6 +326,12 @@ namespace ignite
case SqlState::S08003_NOT_CONNECTED:
return STATE_08003;
+ case SqlState::S08004_CONNECTION_REJECTED:
+ return STATE_08004;
+
+ case SqlState::S08S01_LINK_FAILURE:
+ return STATE_08S01;
+
case SqlState::SHY000_GENERAL_ERROR:
return STATE_HY000;
@@ -269,6 +353,9 @@ namespace ignite
case SqlState::SHY092_OPTION_TYPE_OUT_OF_RANGE:
return STATE_HY092;
+ case SqlState::SHY097_COLUMN_TYPE_OUT_OF_RANGE:
+ return STATE_HY097;
+
case SqlState::SHY105_INVALID_PARAMETER_TYPE:
return STATE_HY105;
@@ -278,7 +365,7 @@ namespace ignite
case SqlState::SHYC00_OPTIONAL_FEATURE_NOT_IMPLEMENTED:
return STATE_HYC00;
- case SqlState::SHYT01_CONNECTIOIN_TIMEOUT:
+ case SqlState::SHYT01_CONNECTION_TIMEOUT:
return STATE_HYT01;
case SqlState::SIM001_FUNCTION_NOT_SUPPORTED:
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/message.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/message.cpp b/modules/platforms/cpp/odbc/src/message.cpp
index 9930290..3601591 100644
--- a/modules/platforms/cpp/odbc/src/message.cpp
+++ b/modules/platforms/cpp/odbc/src/message.cpp
@@ -206,7 +206,9 @@ namespace ignite
writer.WriteObject<std::string>(sqlQuery);
}
- Response::Response(): status(ResponseStatus::FAILED), error()
+ Response::Response() :
+ status(ResponseStatus::UNKNOWN_ERROR),
+ error()
{
// No-op.
}
@@ -216,17 +218,20 @@ namespace ignite
// No-op.
}
- void Response::Read(impl::binary::BinaryReaderImpl& reader)
+ void Response::Read(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver)
{
- status = reader.ReadInt8();
+ if (ver < ProtocolVersion::VERSION_2_1_5)
+ status = reader.ReadInt8();
+ else
+ status = reader.ReadInt32();
if (status == ResponseStatus::SUCCESS)
- ReadOnSuccess(reader);
+ ReadOnSuccess(reader, ver);
else
utility::ReadString(reader, error);;
}
- void Response::ReadOnSuccess(impl::binary::BinaryReaderImpl&)
+ void Response::ReadOnSuccess(impl::binary::BinaryReaderImpl&, const ProtocolVersion&)
{
// No-op.
}
@@ -244,7 +249,7 @@ namespace ignite
// No-op.
}
- void HandshakeResponse::Read(impl::binary::BinaryReaderImpl& reader)
+ void HandshakeResponse::Read(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
accepted = reader.ReadBool();
@@ -270,7 +275,7 @@ namespace ignite
// No-op.
}
- void QueryCloseResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryCloseResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
queryId = reader.ReadInt64();
}
@@ -288,7 +293,7 @@ namespace ignite
// No-op.
}
- void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryExecuteResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
queryId = reader.ReadInt64();
@@ -300,7 +305,8 @@ namespace ignite
QueryExecuteBatchResponse::QueryExecuteBatchResponse():
affectedRows(0),
errorSetIdx(-1),
- errorMessage()
+ errorMessage(),
+ errorCode(1)
{
// No-op.
}
@@ -310,7 +316,7 @@ namespace ignite
// No-op.
}
- void QueryExecuteBatchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryExecuteBatchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion& ver)
{
bool success = reader.ReadBool();
affectedRows = reader.ReadInt64();
@@ -319,6 +325,9 @@ namespace ignite
{
errorSetIdx = reader.ReadInt64();
errorMessage = reader.ReadObject<std::string>();
+
+ if (ver >= ProtocolVersion::VERSION_2_1_5)
+ errorCode = reader.ReadInt32();
}
}
@@ -332,7 +341,7 @@ namespace ignite
// No-op.
}
- void QueryFetchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryFetchResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
queryId = reader.ReadInt64();
@@ -349,7 +358,7 @@ namespace ignite
// No-op.
}
- void QueryGetColumnsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryGetColumnsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
meta::ReadColumnMetaVector(reader, meta);
}
@@ -364,7 +373,7 @@ namespace ignite
// No-op.
}
- void QueryGetTablesMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryGetTablesMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
meta::ReadTableMetaVector(reader, meta);
}
@@ -379,7 +388,7 @@ namespace ignite
// No-op.
}
- void QueryGetParamsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader)
+ void QueryGetParamsMetaResponse::ReadOnSuccess(impl::binary::BinaryReaderImpl& reader, const ProtocolVersion&)
{
utility::ReadByteArray(reader, typeIds);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/query/batch_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/batch_query.cpp b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
index 46447c0..fc8fda4 100644
--- a/modules/platforms/cpp/odbc/src/query/batch_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/batch_query.cpp
@@ -18,6 +18,7 @@
#include "ignite/odbc/connection.h"
#include "ignite/odbc/message.h"
#include "ignite/odbc/log.h"
+#include "ignite/odbc/odbc_error.h"
#include "ignite/odbc/query/batch_query.h"
namespace ignite
@@ -160,9 +161,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- diag.AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -171,7 +178,7 @@ namespace ignite
{
LOG_MSG("Error: " << rsp.GetError());
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
@@ -186,7 +193,7 @@ namespace ignite
setsProcessed += rsp.GetErrorSetIdx();
LOG_MSG("setsProcessed: " << setsProcessed);
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetErrorMessage(),
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetErrorCode()), rsp.GetErrorMessage(),
static_cast<int32_t>(setsProcessed), 0);
return SqlResult::AI_SUCCESS_WITH_INFO;
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp b/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
index b9c08f5..0910612 100644
--- a/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/column_metadata_query.cpp
@@ -21,6 +21,7 @@
#include "ignite/odbc/connection.h"
#include "ignite/odbc/message.h"
#include "ignite/odbc/log.h"
+#include "ignite/odbc/odbc_error.h"
#include "ignite/odbc/query/column_metadata_query.h"
namespace
@@ -297,9 +298,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- diag.AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -307,7 +314,7 @@ namespace ignite
if (rsp.GetStatus() != ResponseStatus::SUCCESS)
{
LOG_MSG("Error: " << rsp.GetError());
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/query/data_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/data_query.cpp b/modules/platforms/cpp/odbc/src/query/data_query.cpp
index 245e0bd..80fcc69 100644
--- a/modules/platforms/cpp/odbc/src/query/data_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/data_query.cpp
@@ -18,6 +18,7 @@
#include "ignite/odbc/connection.h"
#include "ignite/odbc/message.h"
#include "ignite/odbc/log.h"
+#include "ignite/odbc/odbc_error.h"
#include "ignite/odbc/query/data_query.h"
#include "ignite/odbc/query/batch_query.h"
@@ -193,9 +194,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- diag.AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -204,7 +211,7 @@ namespace ignite
{
LOG_MSG("Error: " << rsp.GetError());
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
@@ -241,9 +248,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- diag.AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -254,7 +267,7 @@ namespace ignite
{
LOG_MSG("Error: " << rsp.GetError());
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
@@ -273,9 +286,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- diag.AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -284,7 +303,7 @@ namespace ignite
{
LOG_MSG("Error: " << rsp.GetError());
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp b/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
index e66b281..93f1f79 100644
--- a/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
+++ b/modules/platforms/cpp/odbc/src/query/table_metadata_query.cpp
@@ -21,6 +21,7 @@
#include "ignite/odbc/connection.h"
#include "ignite/odbc/message.h"
#include "ignite/odbc/log.h"
+#include "ignite/odbc/odbc_error.h"
#include "ignite/odbc/query/table_metadata_query.h"
namespace
@@ -223,9 +224,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ diag.AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- diag.AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -234,7 +241,7 @@ namespace ignite
{
LOG_MSG("Error: " << rsp.GetError());
- diag.AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ diag.AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4b4acd39/modules/platforms/cpp/odbc/src/statement.cpp
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/odbc/src/statement.cpp b/modules/platforms/cpp/odbc/src/statement.cpp
index a74a413..36c1a0b 100644
--- a/modules/platforms/cpp/odbc/src/statement.cpp
+++ b/modules/platforms/cpp/odbc/src/statement.cpp
@@ -29,6 +29,7 @@
#include "ignite/odbc/message.h"
#include "ignite/odbc/statement.h"
#include "ignite/odbc/log.h"
+#include "ignite/odbc/odbc_error.h"
namespace ignite
{
@@ -1104,9 +1105,15 @@ namespace ignite
{
connection.SyncMessage(req, rsp);
}
+ catch (const OdbcError& err)
+ {
+ AddStatusRecord(err);
+
+ return SqlResult::AI_ERROR;
+ }
catch (const IgniteError& err)
{
- AddStatusRecord(SqlState::SHYT01_CONNECTIOIN_TIMEOUT, err.GetText());
+ AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, err.GetText());
return SqlResult::AI_ERROR;
}
@@ -1115,7 +1122,7 @@ namespace ignite
{
LOG_MSG("Error: " << rsp.GetError());
- AddStatusRecord(SqlState::SHY000_GENERAL_ERROR, rsp.GetError());
+ AddStatusRecord(ResponseStatusToSqlState(rsp.GetStatus()), rsp.GetError());
return SqlResult::AI_ERROR;
}
[24/35] ignite git commit: ignite-3479
Posted by sb...@apache.org.
ignite-3479
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/65203ce2
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/65203ce2
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/65203ce2
Branch: refs/heads/ignite-3479
Commit: 65203ce26fa5accd53393fcaf1ca27c118183646
Parents: 666a6ac
Author: sboikov <sb...@gridgain.com>
Authored: Fri Sep 22 13:07:58 2017 +0300
Committer: sboikov <sb...@gridgain.com>
Committed: Fri Sep 22 13:07:58 2017 +0300
----------------------------------------------------------------------
.../org/apache/ignite/internal/MvccTestApp.java | 1689 -----------------
.../apache/ignite/internal/MvccTestApp2.java | 1750 ------------------
.../apache/ignite/internal/MvccTestApp3.java | 1713 -----------------
.../cache/IgniteCacheOffheapManagerImpl.java | 3 +
.../mvcc/CacheCoordinatorsSharedManager.java | 82 +-
.../mvcc/MvccCoordinatorVersionResponse.java | 2 +-
.../cache/mvcc/CacheMvccTransactionsTest.java | 38 +-
7 files changed, 92 insertions(+), 5185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/65203ce2/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java b/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java
deleted file mode 100644
index d384339..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/MvccTestApp.java
+++ /dev/null
@@ -1,1689 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.util.GridAtomicLong;
-import org.apache.ignite.internal.util.tostring.GridToStringInclude;
-import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.NotNull;
-import org.jsr166.ConcurrentHashMap8;
-
-/**
- *
- */
-public class MvccTestApp {
- /** */
- private static final boolean DEBUG_LOG = false;
-
- /** */
- private static final boolean SQL = false;
-
- public static void main1(String[] args) throws Exception {
- final MvccTestApp.TestCluster cluster = new MvccTestApp.TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 1, true);
- cluster.txTransfer(0, 2, true);
-
- Map<Object, Object> vals = cluster.sqlAll();
-
- System.out.println();
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)getData.get(i);
-
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main0(String[] args) throws Exception {
- final MvccTestApp.TestCluster cluster = new MvccTestApp.TestCluster(1);
-
- final int ACCOUNTS = 3;
-
- final int START_VAL = 10;
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- cluster.txRemoveTransfer(0, 1);
-
- Map<Object, Object> getData = cluster.sqlAll();;//cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (Map.Entry<Object, Object> e : getData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
-
- System.out.println("Val: " + val);
- }
-
- System.out.println("Sum: " + sum);
-
- cluster.cleanup();
-
- getData = cluster.sqlAll();
-
- System.out.println();
-//
-// MvccQueryVersion ver1 = cluster.crd.queryVersion();
-// MvccQueryVersion ver2 = cluster.crd.queryVersion();
-//
-// cluster.crd.queryDone(ver2.cntr);
-// cluster.crd.queryDone(ver1.cntr);
- }
-
- public static void main(String[] args) throws Exception {
- final AtomicBoolean err = new AtomicBoolean();
-
- final int READ_THREADS = 4;
- final int UPDATE_THREADS = 4;
- final int ACCOUNTS = 50;
-
- final int START_VAL = 100000;
-
- for (int iter = 0; iter < 1000; iter++) {
- System.out.println("Iteration [readThreads=" + READ_THREADS +
- ", updateThreads=" + UPDATE_THREADS + ", accounts=" + ACCOUNTS + ", iter=" + iter + ']');
-
- final TestCluster cluster = new TestCluster(1);
-
- final Map<Object, Object> data = new TreeMap<>();
-
- for (int i = 0; i < ACCOUNTS; i++)
- data.put(i, START_VAL);
-
- cluster.txPutAll(data);
-
- final AtomicBoolean stop = new AtomicBoolean();
-
- List<Thread> threads = new ArrayList<>();
-
- Thread cleanupThread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("cleanup");
-
- try {
- while (!stop.get()) {
- cluster.cleanup();
-
- Thread.sleep(1);
- }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- threads.add(cleanupThread);
-
- cleanupThread.start();
-
- final boolean REMOVES = false;
-
- for (int i = 0; i < READ_THREADS; i++) {
- final int id = i;
-
- Thread thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("read" + id);
-
- int cnt = 0;
-
- while (!stop.get()) {
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- cnt++;
-
- int sum = 0;
-
- if (REMOVES) {
- for (Map.Entry<Object, Object> e : qryData.entrySet()) {
- Integer val = (Integer)e.getValue();
-
- if (val != null)
- sum += val;
- else
- System.out.println("With null");
- }
- }
- else {
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- if (val == null) {
- if (stop.compareAndSet(false, true)) {
- stop.set(true);
- err.set(true);
-
- TestDebugLog.printAllAndExit("No value for key: " + i);
- }
- }
-
- sum += val;
- }
- }
-
- if (sum != ACCOUNTS * START_VAL) {
- if (stop.compareAndSet(false, true)) {
- stop.set(true);
- err.set(true);
-
- TestDebugLog.printAllAndExit("Invalid get sum: " + sum);
- }
- }
-
-// if (cnt % 100 == 0)
-// System.out.println("get " + cnt);
- }
-
- System.out.println("Get cnt: " + cnt);
- }
- });
-
- threads.add(thread);
-
- thread.start();
- }
-
- for (int i = 0; i < UPDATE_THREADS; i++) {
- final int id = i;
-
- Thread thread;
-
- if (REMOVES) {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (rnd.nextBoolean()) {
- cluster.txRemoveTransfer(id1, id2);
- }
- else
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
- else {
- thread = new Thread(new Runnable() {
- @Override public void run() {
- Thread.currentThread().setName("update" + id);
-
- ThreadLocalRandom rnd = ThreadLocalRandom.current();
-
- while (!stop.get()) {
- int id1 = rnd.nextInt(ACCOUNTS);
-
- int id2 = rnd.nextInt(ACCOUNTS);
-
- while (id2 == id1)
- id2 = rnd.nextInt(ACCOUNTS);
-
- if (id1 > id2) {
- int tmp = id1;
- id1 = id2;
- id2 = tmp;
- }
-
- cluster.txTransfer(id1, id2, rnd.nextBoolean());
- }
-
- }
- });
- }
-
- threads.add(thread);
-
- thread.start();
- }
-
- long endTime = System.currentTimeMillis() + 2_000;
-
- while (!stop.get()) {
- Thread.sleep(1000);
-
- if (System.currentTimeMillis() >= endTime)
- break;
-
- //cluster.dumpMvccInfo();
- }
-
- stop.set(true);
-
- for (Thread thread : threads)
- thread.join();
-
- Map<Object, Object> qryData = SQL ? cluster.sqlAll() : cluster.getAll(data.keySet());
-
- int sum = 0;
-
- for (int i = 0; i < ACCOUNTS; i++) {
- Integer val = (Integer)qryData.get(i);
-
- System.out.println("Val " + val);
-
- if (val != null)
- sum += val;
- }
-
- System.out.println("Sum=" + sum + ", expSum=" + (ACCOUNTS * START_VAL));
-
- if (err.get()) {
- System.out.println("Error!");
-
- System.exit(1);
- }
-
-// cluster.dumpMvccInfo();
-//
-// System.out.println("Cleanup");
-//
-// cluster.cleanup();
-//
-// cluster.dumpMvccInfo();
-
- TestDebugLog.clear();
- }
- }
-
- /**
- *
- */
- static class TestCluster {
- /** */
- final List<Node> nodes = new ArrayList<>();
-
- /** */
- final Coordinator crd;
-
- /** */
- final AtomicLong txIdGen = new AtomicLong(10_000);
-
- TestCluster(int nodesNum) {
- crd = new Coordinator();
-
- for (int i = 0; i < nodesNum; i++)
- nodes.add(new Node(i));
- }
-
- void cleanup() {
- CoordinatorCounter cntr = crd.cleanupVersion();
-
- for (Node node : nodes)
- node.dataStore.cleanup(cntr);
- }
-
- void txPutAll(Map<Object, Object> data) {
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- for (Object key : data.keySet()) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), data.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
-
- crd.txDone(txId);
- }
-
- void txTransfer(Integer id1, Integer id2, boolean fromFirst) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(id1);
- keys.add(id2);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer curVal1 = (Integer)vals.get(id1);
- Integer curVal2 = (Integer)vals.get(id2);
-
- boolean update = false;
-
- Integer newVal1 = null;
- Integer newVal2 = null;
-
- if (curVal1 != null && curVal2 != null) {
- if (fromFirst) {
- if (curVal1 > 0) {
- update = true;
-
- newVal1 = curVal1 - 1;
- newVal2 = curVal2 + 1;
- }
- }
- else {
- if (curVal2 > 0) {
- update = true;
-
- newVal1 = curVal1 + 1;
- newVal2 = curVal2 - 1;
- }
- }
- }
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(id1, newVal1);
- newVals.put(id2, newVal2);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("update", txId, id1, newVal1, id2, newVal2, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId);
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- void txRemoveTransfer(Integer from, Integer to) {
- TreeSet<Integer> keys = new TreeSet<>();
-
- keys.add(from);
- keys.add(to);
-
- TxId txId = new TxId(txIdGen.incrementAndGet());
-
- Map<Object, Node> mappedEntries = new LinkedHashMap<>();
-
- Map<Object, Object> vals = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- node.dataStore.lockEntry(key);
-
- vals.put(key, node.dataStore.lastValue(key));
-
- mappedEntries.put(key, node);
- }
-
- CoordinatorCounter cntr = crd.nextTxCounter(txId);
-
- Integer fromVal = (Integer)vals.get(from);
- Integer toVal = (Integer)vals.get(to);
-
- boolean update = fromVal != null && toVal != null;
-
- if (update) {
- Map<Object, Object> newVals = new HashMap<>();
-
- newVals.put(from, null);
- newVals.put(to, fromVal + toVal);
-
- MvccUpdateVersion mvccVer = new MvccUpdateVersion(cntr, txId);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6("remove", txId, from, fromVal, to, toVal, cntr));
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.updateEntry(e.getKey(), newVals.get(e.getKey()), mvccVer);
- }
-
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
- else {
- for (Map.Entry<Object, Node> e : mappedEntries.entrySet()) {
- Node node = e.getValue();
-
- node.dataStore.unlockEntry(e.getKey());
- }
- }
-
- crd.txDone(txId);
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg2("tx done", txId, cntr.cntr));
- }
-
- public void dumpMvccInfo() {
- for (Node node : nodes) {
- int sql = node.dataStore.mvccSqlIdx.size();
-
- for (Map.Entry<Object, MvccValue> e : node.dataStore.mainIdx.entrySet()) {
- List<MvccValue> list = node.dataStore.mvccIdx.get(e.getKey());
-
- int size = 0;
-
- if (list != null) {
- synchronized (list) {
- size = list.size();
- }
- }
-
- System.out.println("Mvcc info [key=" + e.getKey() +
- ", val=" + e.getValue() +
- ", mvccVals=" + size +
- ", sqlVals=" + sql + ']');
- }
- }
- }
-
- public Map<Object, Object> sqlAll() {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Node node : nodes) {
- Map<Object, Object> nodeRes = node.dataStore.sqlQuery(qryVer);
-
- res.putAll(nodeRes);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sqlAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- public Map<Object, Object> getAll(Set<?> keys) {
- MvccQueryVersion qryVer = crd.queryVersion();
-
- Map<Object, Object> res = new HashMap<>();
-
- for (Object key : keys) {
- int nodeIdx = nodeForKey(key);
-
- Node node = nodes.get(nodeIdx);
-
- Object val = node.dataStore.get(key, qryVer);
-
- res.put(key, val);
- }
-
- crd.queryDone(qryVer.cntr);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("getAll", qryVer.cntr, qryVer.activeTxs, res));
- }
-
- return res;
- }
-
- private int nodeForKey(Object key) {
- return U.safeAbs(key.hashCode()) % nodes.size();
- }
- }
-
- /**
- *
- */
- static class Node {
- /** */
- final DataStore dataStore;
-
- /** */
- final int nodexIdx;
-
- public Node(int nodexIdx) {
- this.nodexIdx = nodexIdx;
-
- dataStore = new DataStore();
- }
-
- @Override public String toString() {
- return "Node [idx=" + nodexIdx + ']';
- }
- }
-
- /**
- *
- */
- static class Coordinator {
- /** */
- private final AtomicLong cntr = new AtomicLong(-1);
-
- /** */
- private final GridAtomicLong commitCntr = new GridAtomicLong(-1);
-
- /** */
- private final ConcurrentHashMap8<Long, QueryCounter> activeQueries = new ConcurrentHashMap8<>();
-
- /** */
- @GridToStringInclude
- private final ConcurrentHashMap8<TxId, TxId> activeTxs = new ConcurrentHashMap8<>();
-
- CoordinatorCounter nextTxCounter(TxId txId) {
- activeTxs.put(txId, txId);
-
- CoordinatorCounter newCtr = new CoordinatorCounter(cntr.incrementAndGet());
-
- txId.cntr = newCtr.cntr;
-
- return newCtr;
- }
-
- void txDone(TxId txId) {
- TxId cntr = activeTxs.remove(txId);
-
- assert cntr != null && cntr.cntr != -1L;
-
- commitCntr.setIfGreater(cntr.cntr);
- }
-
- private Long minActive(Set<TxId> txs) {
- Long minActive = null;
-
- for (Map.Entry<TxId, TxId> e : activeTxs.entrySet()) {
- if (txs != null)
- txs.add(e.getKey());
-
- TxId val = e.getValue();
-
- while (val.cntr == -1)
- Thread.yield();
-
- long cntr = val.cntr;
-
- if (minActive == null)
- minActive = cntr;
- else if (cntr < minActive)
- minActive = cntr;
- }
-
- return minActive;
- }
-
- static class QueryCounter extends AtomicInteger {
- public QueryCounter(int initialValue) {
- super(initialValue);
- }
-
- boolean increment2() {
- for (;;) {
- int current = get();
- int next = current + 1;
-
- if (current == 0)
- return false;
-
- if (compareAndSet(current, next))
- return true;
- }
- }
- }
-
- private ReadWriteLock rwLock = new ReentrantReadWriteLock();
-
- MvccQueryVersion queryVersion() {
- rwLock.readLock().lock();
-
- long useCntr = commitCntr.get();
-
- Set<TxId> txs = new HashSet<>();
-
- Long minActive = minActive(txs);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive;
-
- MvccQueryVersion qryVer = new MvccQueryVersion(new CoordinatorCounter(useCntr), txs);
-
- for (;;) {
- QueryCounter qryCnt = activeQueries.get(useCntr);
-
- if (qryCnt != null) {
- boolean inc = qryCnt.increment2();
-
- if (!inc) {
- activeQueries.remove(useCntr, qryCnt);
-
- continue;
- }
- }
- else {
- qryCnt = new QueryCounter(1);
-
- if (activeQueries.putIfAbsent(useCntr, qryCnt) != null)
- continue;
- }
-
- break;
- }
-
- rwLock.readLock().unlock();
-
- return qryVer;
- }
-
- void queryDone(CoordinatorCounter cntr) {
- AtomicInteger qryCnt = activeQueries.get(cntr.cntr);
-
- assert qryCnt != null : cntr.cntr;
-
- int left = qryCnt.decrementAndGet();
-
- assert left >= 0 : left;
-
- if (left == 0)
- activeQueries.remove(cntr.cntr, qryCnt);
- }
-
- CoordinatorCounter cleanupVersion() {
- rwLock.writeLock().lock();
-
- long useCntr = commitCntr.get();
-
- Long minActive = minActive(null);
-
- if (minActive != null && minActive < useCntr)
- useCntr = minActive - 1;
-
- for (Long qryCntr : activeQueries.keySet()) {
- if (qryCntr <= useCntr)
- useCntr = qryCntr - 1;
- }
-
- rwLock.writeLock().unlock();
-
- return new CoordinatorCounter(useCntr);
- }
-
- @Override public String toString() {
- return S.toString(Coordinator.class, this);
- }
- }
-
- /**
- *
- */
- static class CoordinatorCounter implements Comparable<CoordinatorCounter> {
- /** */
- private final long topVer; // TODO
-
- /** */
- private final long cntr;
-
- CoordinatorCounter(long cntr) {
- this.topVer = 1;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(CoordinatorCounter o) {
- return Long.compare(cntr, o.cntr);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- CoordinatorCounter that = (CoordinatorCounter)o;
-
- return cntr == that.cntr;
- }
-
- @Override public int hashCode() {
- return (int)(cntr ^ (cntr >>> 32));
- }
-
- @Override public String toString() {
- return "Cntr [c=" + cntr + ']';
- }
- }
-
- /**
- *
- */
- static class MvccUpdateVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final TxId txId;
-
- /**
- * @param cntr
- */
- MvccUpdateVersion(CoordinatorCounter cntr, TxId txId) {
- assert cntr != null;
-
- this.cntr = cntr;
- this.txId = txId;
- }
-
- @Override public String toString() {
- return S.toString(MvccUpdateVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccQueryVersion {
- /** */
- @GridToStringInclude
- final CoordinatorCounter cntr;
-
- /** */
- @GridToStringInclude
- final Collection<TxId> activeTxs;
-
- MvccQueryVersion(CoordinatorCounter cntr, Collection<TxId> activeTxs) {
- this.cntr = cntr;
- this.activeTxs = activeTxs;
- }
-
- @Override public String toString() {
- return S.toString(MvccQueryVersion.class, this);
- }
- }
-
- /**
- *
- */
- static class TxId {
- long cntr = -1;
-
- /** */
- @GridToStringInclude
- final long id;
-
- TxId(long id) {
- this.id = id;
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- TxId txId = (TxId) o;
-
- return id == txId.id;
- }
-
- @Override public int hashCode() {
- return (int) (id ^ (id >>> 32));
- }
-
- @Override public String toString() {
- return S.toString(TxId.class, this);
- }
- }
-
- /**
- *
- */
- static class SqlKey implements Comparable<SqlKey> {
- /** */
- final Comparable key;
-
- /** */
- final Comparable val;
-
- /** */
- final CoordinatorCounter cntr;
-
- public SqlKey(Object key, Object val, CoordinatorCounter cntr) {
- this.key = (Comparable)key;
- this.val = (Comparable)val;
- this.cntr = cntr;
- }
-
- @Override public int compareTo(@NotNull SqlKey o) {
- int cmp;
-
- if (val != null && o.val != null)
- cmp = val.compareTo(o.val);
- else {
- if (val != null)
- cmp = 1;
- else
- cmp = o.val == null ? 0 : -1;
- }
-
-
- if (cmp == 0) {
- cmp = key.compareTo(o.key);
-
- if (cmp == 0)
- cmp = cntr.compareTo(o.cntr);
- }
-
- return cmp;
- }
-
- @Override public String toString() {
- return "SqlKey [key=" + key + ", val=" + val + ']';
- }
- }
-
- /**
- *
- */
- static class DataStore {
- /** */
- private final ConcurrentHashMap<Object, ReentrantLock> locks = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, MvccValue> mainIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentHashMap<Object, List<MvccValue>> mvccIdx = new ConcurrentHashMap<>();
-
- /** */
- final ConcurrentSkipListMap<SqlKey, MvccSqlValue> mvccSqlIdx = new ConcurrentSkipListMap<>();
-
- void cleanup(CoordinatorCounter cleanupCntr) {
- for (Map.Entry<Object, List<MvccValue>> e : mvccIdx.entrySet()) {
- lockEntry(e.getKey());
-
- try {
- List<MvccValue> list = e.getValue();
-
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (val.ver.cntr.compareTo(cleanupCntr) <= 0) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg6_1("cleanup",
- e.getKey(), val.val, val.ver, cleanupCntr.cntr, null, null));
- }
-
- MvccValue prev;
-
- if (val.val != null)
- prev = mainIdx.put(e.getKey(), val);
- else
- prev = mainIdx.remove(e.getKey());
-
- if (prev != null) {
- SqlKey key = new SqlKey(e.getKey(), prev.val, prev.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
-
- for (int j = 0; j <= i; j++) {
- MvccValue rmvd = list.remove(0);
-
- assert rmvd != null;
-
- if (j != i || rmvd.val == null) {
- SqlKey key = new SqlKey(e.getKey(), rmvd.val, rmvd.ver.cntr);
-
- MvccSqlValue old = mvccSqlIdx.remove(key);
-
- assert old != null;
- }
- }
-
- if (list.isEmpty())
- mvccIdx.remove(e.getKey());
-
- break;
- }
- }
- }
- }
- finally {
- unlockEntry(e.getKey());
- }
- }
- }
-
- void lockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.lock();
- }
-
- void unlockEntry(Object key) {
- ReentrantLock e = lock(key);
-
- e.unlock();
- }
-
- void updateEntry(Object key, Object val, MvccUpdateVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list == null) {
- Object old = mvccIdx.putIfAbsent(key, list = new ArrayList<>());
-
- assert old == null;
- }
-
- MvccValue prevVal = null;
-
- synchronized (list) {
- if (!list.isEmpty())
- prevVal = list.get(list.size() - 1);
-
- list.add(new MvccValue(val, ver));
- }
-
- if (prevVal == null)
- prevVal = mainIdx.get(key);
-
- if (prevVal != null) {
- SqlKey prevKey = new SqlKey(key, prevVal.val, prevVal.ver.cntr);
-
- MvccSqlValue old =
- mvccSqlIdx.put(prevKey, new MvccSqlValue(prevVal.val, prevVal.ver, ver));
-
- assert old != null;
- }
-
- mvccSqlIdx.put(new SqlKey(key, val, ver.cntr), new MvccSqlValue(val, ver, null));
- }
-
- Object lastValue(Object key) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- if (list.size() > 0)
- return list.get(list.size() - 1).val;
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- return val != null ? val.val : null;
- }
-
- Map<Object, Object> sqlQuery(MvccQueryVersion qryVer) {
- Map<Object, Object> res = new HashMap<>();
-
- for (Map.Entry<SqlKey, MvccSqlValue> e : mvccSqlIdx.entrySet()) {
- MvccSqlValue val = e.getValue();
-
- if (!versionVisible(val.ver, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("sql skip mvcc val", e.getKey().key, val.val, val.ver));
- }
-
- continue;
- }
-
- MvccUpdateVersion newVer = val.newVer;
-
- if (newVer != null && versionVisible(newVer, qryVer)) {
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql skip mvcc val2", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- continue;
- }
-
- Object old = res.put(e.getKey().key, e.getValue().val);
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg4("sql get mvcc val", e.getKey().key, val.val, val.ver, val.newVer));
- }
-
- if (old != null) {
- TestDebugLog.printAllAndExit("Already has value for key [key=" + e.getKey().key +
- ", qryVer=" + qryVer +
- ", oldVal=" + old +
- ", newVal=" + e.getValue().val +
- ']');
- }
-
- assert old == null;
- }
-
- return res;
- }
-
- private boolean versionVisible(MvccUpdateVersion ver, MvccQueryVersion qryVer) {
- int cmp = ver.cntr.compareTo(qryVer.cntr);
-
- return cmp <= 0 && !qryVer.activeTxs.contains(ver.txId);
- }
-
- Object get(Object key, MvccQueryVersion ver) {
- List<MvccValue> list = mvccIdx.get(key);
-
- if (list != null) {
- synchronized (list) {
- for (int i = list.size() - 1; i >= 0; i--) {
- MvccValue val = list.get(i);
-
- if (!versionVisible(val.ver, ver))
- continue;
-
- if (DEBUG_LOG) {
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read mvcc val", key, val, val.ver));
- }
-
- return val.val;
- }
- }
- }
-
- MvccValue val = mainIdx.get(key);
-
- if (val != null) {
- int cmp = val.ver.cntr.compareTo(ver.cntr);
-
- assert cmp <= 0 : "Committed [ver=" + val.ver + ", qryVer=" + ver.cntr + ']';
-
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted val", key, val, val.ver));
- }
- else {
- if (DEBUG_LOG)
- TestDebugLog.msgs.add(new TestDebugLog.Msg3("read comitted null", key, null, null));
- }
-
- return val != null ? val.val : null;
- }
-
- private ReentrantLock lock(Object key) {
- ReentrantLock e = locks.get(key);
-
- if (e == null) {
- ReentrantLock old = locks.putIfAbsent(key, e = new ReentrantLock());
-
- if (old != null)
- e = old;
- }
-
- return e;
- }
- }
-
- /**
- *
- */
- static class MvccValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- MvccValue(Object val, MvccUpdateVersion ver) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- }
-
- @Override public String toString() {
- return S.toString(MvccValue.class, this);
- }
- }
-
- /**
- *
- */
- static class MvccSqlValue {
- /** */
- @GridToStringInclude
- final Object val;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion ver;
-
- /** */
- @GridToStringInclude
- final MvccUpdateVersion newVer;
-
- MvccSqlValue(Object val, MvccUpdateVersion ver, MvccUpdateVersion newVer) {
- assert ver != null;
-
- this.val = val;
- this.ver = ver;
- this.newVer = newVer;
- }
-
- @Override public String toString() {
- return S.toString(MvccSqlValue.class, this);
- }
- }
-
- static void log(String msg) {
- System.out.println(Thread.currentThread() + ": " + msg);
- }
-}
-
-class TestDebugLog {
- /** */
- static final List<Object> msgs = Collections.synchronizedList(new ArrayList<>(100_000));
-
- /** */
- private static final SimpleDateFormat DEBUG_DATE_FMT = new SimpleDateFormat("HH:mm:ss,SSS");
-
- static class Message {
- String thread = Thread.currentThread().getName();
-
- String msg;
-
- long ts = U.currentTimeMillis();
-
- public Message(String msg) {
- this.msg = msg;
- }
-
- public String toString() {
- return "Msg [msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg2 extends Message{
- Object v1;
- Object v2;
-
- public Msg2(String msg, Object v1, Object v2) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", msg=" + msg +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg3 extends Message{
- Object v1;
- Object v2;
- Object v3;
-
- public Msg3(String msg, Object v1, Object v2, Object v3) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- }
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg4 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
-
- public Msg4(String msg, Object v1, Object v2, Object v3, Object v4) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", v1=" + v1 +
- ", v2=" + v2 +
- ", v3=" + v3 +
- ", v4=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class Msg6 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", txId=" + v1 +
- ", id1=" + v2 +
- ", v1=" + v3 +
- ", id2=" + v4 +
- ", v2=" + v5 +
- ", cntr=" + v6 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
- static class Msg6_1 extends Message{
- Object v1;
- Object v2;
- Object v3;
- Object v4;
- Object v5;
- Object v6;
-
- public Msg6_1(String msg, Object v1, Object v2, Object v3, Object v4, Object v5, Object v6) {
- super(msg);
- this.v1 = v1;
- this.v2 = v2;
- this.v3 = v3;
- this.v4 = v4;
- this.v5 = v5;
- this.v6 = v6;
- }
-
- public String toString() {
- return "Msg [msg=" + msg +
- ", key=" + v1 +
- ", val=" + v2 +
- ", ver=" + v3 +
- ", cleanupC=" + v4 +
- ", thread=" + thread +
- ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class EntryMessage extends Message {
- Object key;
- Object val;
-
- public EntryMessage(Object key, Object val, String msg) {
- super(msg);
-
- this.key = key;
- this.val = val;
- }
-
- public String toString() {
- return "EntryMsg [key=" + key + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static class PartMessage extends Message {
- int p;
- Object val;
-
- public PartMessage(int p, Object val, String msg) {
- super(msg);
-
- this.p = p;
- this.val = val;
- }
-
- public String toString() {
- return "PartMessage [p=" + p + ", val=" + val + ", msg=" + msg + ", thread=" + thread + ", time=" + DEBUG_DATE_FMT.format(new Date(ts)) + ']';
- }
- }
-
- static final boolean out = false;
-
- public static void addMessage(String msg) {
- msgs.add(new Message(msg));
-
- if (out)
- System.out.println(msg);
- }
-
- public static void addEntryMessage(Object key, Object val, String msg) {
- if (key instanceof KeyCacheObject)
- key = ((KeyCacheObject)key).value(null, false);
-
- EntryMessage msg0 = new EntryMessage(key, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- public static void addPartMessage(int p, Object val, String msg) {
- PartMessage msg0 = new PartMessage(p, val, msg);
-
- msgs.add(msg0);
-
- if (out) {
- System.out.println(msg0.toString());
-
- System.out.flush();
- }
- }
-
- static void printAllAndExit(String msg) {
- System.out.println(msg);
-
- TestDebugLog.addMessage(msg);
-
- List<Object> msgs = TestDebugLog.printMessages(true, null);
-
- TestDebugLog.printMessages0(msgs, "test_debug_update.txt");
-
- TestDebugLog.printMessagesForThread(msgs, Thread.currentThread().getName(), "test_debug_thread.txt");
-
- System.exit(1);
- }
-
- public static void printMessagesForThread(List<Object> msgs0, String thread0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String thread = ((Message) msg).thread;
-
- if (thread.equals(thread0))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static void printMessages0(List<Object> msgs0, String file) {
- try {
- FileOutputStream out = new FileOutputStream(file);
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof Message) {
- String msg0 = ((Message) msg).msg;
-
- if (msg0.equals("tx done") || msg0.equals("update") || msg0.equals("cleanup"))
- w.println(msg.toString());
- }
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public static List<Object> printMessages(boolean file, Integer part) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (part != null && msg instanceof PartMessage) {
- if (((PartMessage) msg).p != part)
- continue;
- }
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0)
- System.out.println(msg);
- }
-
- return msgs0;
- }
-
- public static void printKeyMessages(boolean file, Object key) {
- List<Object> msgs0;
-
- synchronized (msgs) {
- msgs0 = new ArrayList<>(msgs);
-
- msgs.clear();
- }
-
- if (file) {
- try {
- FileOutputStream out = new FileOutputStream("test_debug.log");
-
- PrintWriter w = new PrintWriter(out);
-
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- w.println(msg.toString());
- }
-
- w.close();
-
- out.close();
- }
- catch (IOException e) {
- e.printStackTrace();
- }
- }
- else {
- for (Object msg : msgs0) {
- if (msg instanceof EntryMessage && !((EntryMessage)msg).key.equals(key))
- continue;
-
- System.out.println(msg);
- }
- }
- }
-
- public static void clear() {
- msgs.clear();
- }
-
- public static void clearEntries() {
- for (Iterator it = msgs.iterator(); it.hasNext();) {
- Object msg = it.next();
-
- if (msg instanceof EntryMessage)
- it.remove();
- }
- }
-
-}
\ No newline at end of file
[28/35] ignite git commit: IGNITE-6334 Fixed compilation after
conflict
Posted by sb...@apache.org.
IGNITE-6334 Fixed compilation after conflict
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e3d448e6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e3d448e6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e3d448e6
Branch: refs/heads/ignite-3479
Commit: e3d448e6c23df67b15f8cf7545b4754807acac32
Parents: 9f662cc
Author: Alexey Goncharuk <al...@gmail.com>
Authored: Fri Sep 22 13:12:15 2017 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 22 13:12:31 2017 +0300
----------------------------------------------------------------------
.../persistence/pagemem/PagesWriteThrottleSmokeTest.java | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e3d448e6/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
index 12a601d..f4f21ef 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PagesWriteThrottleSmokeTest.java
@@ -20,6 +20,7 @@ import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.nio.ByteBuffer;
+import java.nio.file.OpenOption;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.LockSupport;
@@ -50,6 +51,10 @@ import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.READ;
+import static java.nio.file.StandardOpenOption.WRITE;
+
/**
*
*/
@@ -288,12 +293,12 @@ public class PagesWriteThrottleSmokeTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override public FileIO create(File file) throws IOException {
- return create(file, "rw");
+ return create(file, CREATE, READ, WRITE);
}
/** {@inheritDoc} */
- @Override public FileIO create(File file, String mode) throws IOException {
- final FileIO delegate = delegateFactory.create(file, mode);
+ @Override public FileIO create(File file, OpenOption... openOption) throws IOException {
+ final FileIO delegate = delegateFactory.create(file, openOption);
return new FileIODecorator(delegate) {
@Override public int write(ByteBuffer srcBuf) throws IOException {
[17/35] ignite git commit: ignite-6181 Tx rollback on timeout
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 9a8280f..77634bd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -291,7 +291,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
public void rollbackTransactionsForCache(int cacheId) {
rollbackTransactionsForCache(cacheId, nearIdMap);
- rollbackTransactionsForCache(cacheId, threadMap);
+ rollbackTransactionsForCache(cacheId, idMap);
}
/**
@@ -304,7 +304,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
for (IgniteTxEntry entry : tx.allEntries()) {
if (entry.cacheId() == cacheId) {
- rollbackTx(tx);
+ rollbackTx(tx, false);
break;
}
@@ -316,8 +316,10 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
@Override public void onDisconnected(IgniteFuture reconnectFut) {
txFinishSync.onDisconnected(reconnectFut);
- for (Map.Entry<Long, IgniteInternalTx> e : threadMap.entrySet())
- rollbackTx(e.getValue());
+ for (IgniteInternalTx tx : idMap.values())
+ rollbackTx(tx, true);
+ for (IgniteInternalTx tx : nearIdMap.values())
+ rollbackTx(tx, true);
IgniteClientDisconnectedException err =
new IgniteClientDisconnectedException(reconnectFut, "Client node disconnected.");
@@ -378,6 +380,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
X.println(">>> Transaction manager memory stats [igniteInstanceName=" + cctx.igniteInstanceName() + ']');
X.println(">>> threadMapSize: " + threadMap.size());
X.println(">>> idMap [size=" + idMap.size() + ']');
+ X.println(">>> nearIdMap [size=" + nearIdMap.size() + ']');
X.println(">>> completedVersSortedSize: " + completedVersSorted.size());
X.println(">>> completedVersHashMapSize: " + completedVersHashMap.sizex());
}
@@ -490,14 +493,15 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
IgniteInternalTx t;
if ((t = txIdMap.putIfAbsent(tx.xidVersion(), tx)) == null) {
- // Add both, explicit and implicit transactions.
- // Do not add remote and dht local transactions as remote node may have the same thread ID
- // and overwrite local transaction.
if (tx.local() && !tx.dht()) {
- if (cacheCtx == null || !cacheCtx.systemTx())
- threadMap.put(tx.threadId(), tx);
- else
- sysThreadMap.put(new TxThreadKey(tx.threadId(), cacheCtx.cacheId()), tx);
+ assert tx instanceof GridNearTxLocal : tx;
+
+ if (!tx.implicit()) {
+ if (cacheCtx == null || !cacheCtx.systemTx())
+ threadMap.put(tx.threadId(), tx);
+ else
+ sysThreadMap.put(new TxThreadKey(tx.threadId(), cacheCtx.cacheId()), tx);
+ }
}
// Handle mapped versions.
@@ -633,11 +637,10 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
/**
* @return Local transaction.
*/
- @SuppressWarnings({"unchecked"})
- @Nullable public <T> T localTx() {
- IgniteInternalTx tx = tx();
+ @Nullable public IgniteTxLocalAdapter localTx() {
+ IgniteTxLocalAdapter tx = tx();
- return tx != null && tx.local() ? (T)tx : null;
+ return tx != null && tx.local() ? tx : null;
}
/**
@@ -719,15 +722,6 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
}
/**
- * @return Local transaction.
- */
- @Nullable public IgniteInternalTx localTxx() {
- IgniteInternalTx tx = tx();
-
- return tx != null && tx.local() ? tx : null;
- }
-
- /**
* @return User transaction for current thread.
*/
@Nullable public GridNearTxLocal userTx() {
@@ -1215,32 +1209,32 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
collectPendingVersions(dhtTxLoc);
}
- // 4. Unlock write resources.
+ // 3. Unlock write resources.
unlockMultiple(tx, tx.writeEntries());
- // 5. Unlock read resources if required.
+ // 4. Unlock read resources if required.
if (unlockReadEntries(tx))
unlockMultiple(tx, tx.readEntries());
- // 6. Notify evictions.
+ // 5. Notify evictions.
notifyEvictions(tx);
- // 7. Remove obsolete entries from cache.
+ // 6. Remove obsolete entries from cache.
removeObsolete(tx);
- // 8. Assign transaction number at the end of transaction.
+ // 7. Assign transaction number at the end of transaction.
tx.endVersion(cctx.versions().next(tx.topologyVersion()));
- // 9. Remove from per-thread storage.
+ // 8. Remove from per-thread storage.
clearThreadMap(tx);
- // 10. Unregister explicit locks.
+ // 9. Unregister explicit locks.
if (!tx.alternateVersions().isEmpty()) {
for (GridCacheVersion ver : tx.alternateVersions())
idMap.remove(ver);
}
- // 11. Remove Near-2-DHT mappings.
+ // 10. Remove Near-2-DHT mappings.
if (tx instanceof GridCacheMappedVersion) {
GridCacheVersion mapped = ((GridCacheMappedVersion)tx).mappedVersion();
@@ -1248,10 +1242,10 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
mappedVers.remove(mapped);
}
- // 12. Clear context.
+ // 11. Clear context.
resetContext();
- // 14. Update metrics.
+ // 12. Update metrics.
if (!tx.dht() && tx.local()) {
if (!tx.system())
cctx.txMetrics().onTxCommit();
@@ -1275,8 +1269,9 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* Rolls back a transaction.
*
* @param tx Transaction to rollback.
+ * @param clearThreadMap {@code True} if need remove tx from thread map.
*/
- public void rollbackTx(IgniteInternalTx tx) {
+ public void rollbackTx(IgniteInternalTx tx, boolean clearThreadMap) {
assert tx != null;
if (log.isDebugEnabled())
@@ -1302,7 +1297,8 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
removeObsolete(tx);
// 6. Remove from per-thread storage.
- clearThreadMap(tx);
+ if (clearThreadMap)
+ clearThreadMap(tx);
// 7. Unregister explicit locks.
if (!tx.alternateVersions().isEmpty())
@@ -1427,8 +1423,10 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
/**
* @param tx Transaction to clear.
*/
- private void clearThreadMap(IgniteInternalTx tx) {
+ public void clearThreadMap(IgniteInternalTx tx) {
if (tx.local() && !tx.dht()) {
+ assert tx instanceof GridNearTxLocal : tx;
+
if (!tx.system())
threadMap.remove(tx.threadId(), tx);
else {
@@ -2257,6 +2255,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* @see #resumeTx(GridNearTxLocal)
* @see GridNearTxLocal#suspend()
* @see GridNearTxLocal#resume()
+ * @throws IgniteCheckedException If failed to suspend transaction.
*/
public void suspendTx(final GridNearTxLocal tx) throws IgniteCheckedException {
assert tx != null && !tx.system() : tx;
@@ -2280,6 +2279,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* @see #suspendTx(GridNearTxLocal)
* @see GridNearTxLocal#suspend()
* @see GridNearTxLocal#resume()
+ * @throws IgniteCheckedException If failed to resume tx.
*/
public void resumeTx(GridNearTxLocal tx) throws IgniteCheckedException {
assert tx != null && !tx.system() : tx;
@@ -2287,7 +2287,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
assert !transactionMap(tx).containsValue(tx) : tx;
assert !haveSystemTxForThread(Thread.currentThread().getId());
- if(!tx.state(ACTIVE)) {
+ if (!tx.state(ACTIVE)) {
throw new IgniteCheckedException("Trying to resume transaction with incorrect state "
+ "[expected=" + SUSPENDED + ", actual=" + tx.state() + ']');
}
@@ -2295,10 +2295,10 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
long threadId = Thread.currentThread().getId();
if (threadMap.putIfAbsent(threadId, tx) != null)
- throw new IgniteCheckedException("Thread already start a transaction.");
+ throw new IgniteCheckedException("Thread already has started a transaction.");
if (transactionMap(tx).putIfAbsent(tx.xidVersion(), tx) != null)
- throw new IgniteCheckedException("Thread already start a transaction.");
+ throw new IgniteCheckedException("Thread already has started a transaction.");
tx.threadId(threadId);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
index 8c71f76..ff6beb4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/timeout/GridTimeoutProcessor.java
@@ -84,12 +84,13 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
/**
* @param timeoutObj Timeout object.
+ * @return {@code True} if object was added.
*/
@SuppressWarnings({"NakedNotify", "CallToNotifyInsteadOfNotifyAll"})
- public void addTimeoutObject(GridTimeoutObject timeoutObj) {
+ public boolean addTimeoutObject(GridTimeoutObject timeoutObj) {
if (timeoutObj.endTime() <= 0 || timeoutObj.endTime() == Long.MAX_VALUE)
// Timeout will never happen.
- return;
+ return false;
boolean added = timeoutObjs.add(timeoutObj);
@@ -100,6 +101,8 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
mux.notify(); // No need to notifyAll since we only have one thread.
}
}
+
+ return true;
}
/**
@@ -124,9 +127,10 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
/**
* @param timeoutObj Timeout object.
+ * @return {@code True} if timeout object was removed.
*/
- public void removeTimeoutObject(GridTimeoutObject timeoutObj) {
- timeoutObjs.remove(timeoutObj);
+ public boolean removeTimeoutObject(GridTimeoutObject timeoutObj) {
+ return timeoutObjs.remove(timeoutObj);
}
/**
@@ -149,13 +153,14 @@ public class GridTimeoutProcessor extends GridProcessorAdapter {
GridTimeoutObject timeoutObj = iter.next();
if (timeoutObj.endTime() <= now) {
- iter.remove();
+ try {
+ boolean rmvd = timeoutObjs.remove(timeoutObj);
- if (log.isDebugEnabled())
- log.debug("Timeout has occurred: " + timeoutObj);
+ if (log.isDebugEnabled())
+ log.debug("Timeout has occurred [obj=" + timeoutObj + ", process=" + rmvd + ']');
- try {
- timeoutObj.onTimeout();
+ if (rmvd)
+ timeoutObj.onTimeout();
}
catch (Throwable e) {
if (isCancelled() && !(e instanceof Error)){
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java
index 9f4910a..79316bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/CacheTxFastFinishTest.java
@@ -24,6 +24,7 @@ import org.apache.ignite.IgniteTransactions;
import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFastFinishFuture;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
@@ -210,8 +211,7 @@ public class CacheTxFastFinishTest extends GridCommonAbstractTest {
IgniteInternalTx tx0 = ((TransactionProxyImpl)tx).tx();
assertNull(fieldValue(tx0, "prepFut"));
- assertNull(fieldValue(tx0, "commitFut"));
- assertNull(fieldValue(tx0, "rollbackFut"));
+ assertTrue(fieldValue(tx0, "finishFut") instanceof GridNearTxFastFinishFuture);
}
/**
@@ -225,12 +225,13 @@ public class CacheTxFastFinishTest extends GridCommonAbstractTest {
tx.commit();
assertNotNull(fieldValue(tx0, "prepFut"));
- assertNotNull(fieldValue(tx0, "commitFut"));
+ assertNotNull(fieldValue(tx0, "finishFut"));
}
else {
tx.rollback();
- assertNotNull(fieldValue(tx0, "rollbackFut"));
+ assertNull(fieldValue(tx0, "prepFut"));
+ assertNotNull(fieldValue(tx0, "finishFut"));
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java
index 2efa0cb..f2e17e4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/IgniteTxConfigCacheSelfTest.java
@@ -185,6 +185,8 @@ public class IgniteTxConfigCacheSelfTest extends GridCommonAbstractTest {
try (final Transaction tx = ignite.transactions().txStart()) {
assert tx != null;
+ cache.put("key0", "val0");
+
sleepForTxFailure();
cache.put("key", "val");
@@ -195,7 +197,19 @@ public class IgniteTxConfigCacheSelfTest extends GridCommonAbstractTest {
assert e.getCause() instanceof TransactionTimeoutException;
}
+ assertNull(ignite.transactions().tx());
+
+ assert !cache.containsKey("key0");
assert !cache.containsKey("key");
+
+ // New transaction must succeed.
+ try (final Transaction tx = ignite.transactions().txStart()) {
+ cache.put("key", "val");
+
+ tx.commit();
+ }
+
+ assert cache.containsKey("key");
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java
new file mode 100644
index 0000000..c8eac20
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheThreadLocalTxTest.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed;
+
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.internal.IgniteKernal;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.lang.IgniteFuture;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+
+/**
+ *
+ */
+public class IgniteCacheThreadLocalTxTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private boolean client;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(ipFinder);
+
+ cfg.setClientMode(client);
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ super.afterTest();
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testSingleNode() throws Exception {
+ threadLocalTx(startGrid(0));
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testMultiNode() throws Exception {
+ startGridsMultiThreaded(4);
+
+ client = true;
+
+ startGrid(4);
+
+ for (Ignite node : G.allGrids())
+ threadLocalTx(node);
+ }
+
+ /**
+ * @param node Node.
+ * @throws Exception If failed.
+ */
+ @SuppressWarnings("unchecked")
+ private void threadLocalTx(Ignite node) throws Exception {
+ CacheConfiguration ccfg = new CacheConfiguration(DEFAULT_CACHE_NAME);
+
+ ccfg.setAtomicityMode(TRANSACTIONAL);
+ ccfg.setBackups(2);
+
+ IgniteCache<Object, Object> cache = node.getOrCreateCache(ccfg);
+
+ checkNoTx(node);
+
+ boolean[] reads = {true, false};
+ boolean[] writes = {true, false};
+ int endOps = 5;
+
+ for (TransactionConcurrency concurrency : TransactionConcurrency.values()) {
+ for (TransactionIsolation isolation : TransactionIsolation.values()) {
+ for (boolean read : reads) {
+ for (boolean write : writes) {
+ for (int i = 0; i < endOps; i++)
+ checkTx(concurrency, isolation, node, cache, read, write, i);
+ }
+ }
+ }
+ }
+
+ checkNoTx(node);
+
+ cache.put(1, 1);
+
+ checkNoTx(node);
+ }
+
+ /**
+ * @param concurrency Tx concurrency.
+ * @param isolation Tx isolation.
+ * @param node Node.
+ * @param cache Cache.
+ * @param read {@code True} if read in tx.
+ * @param write {@code True} if write in tx.
+ * @param endOp Operation to test.
+ */
+ private void checkTx(TransactionConcurrency concurrency,
+ TransactionIsolation isolation,
+ Ignite node,
+ IgniteCache<Object, Object> cache,
+ boolean read,
+ boolean write,
+ int endOp) {
+ IgniteTransactions txs = node.transactions();
+
+ checkNoTx(node);
+
+ Transaction tx = txs.txStart(concurrency, isolation);
+
+ assertEquals(tx, txs.tx());
+
+ try {
+ txs.txStart(concurrency, isolation);
+
+ fail();
+ }
+ catch (IllegalStateException expected) {
+ // No-op.
+ }
+
+ if (read)
+ cache.get(ThreadLocalRandom.current().nextInt(100_000));
+
+ if (write)
+ cache.put(ThreadLocalRandom.current().nextInt(100_000), 1);
+
+
+ try {
+ txs.txStart(concurrency, isolation);
+
+ fail();
+ }
+ catch (IllegalStateException expected) {
+ // No-op.
+ }
+
+ assertEquals(tx, txs.tx());
+
+ IgniteFuture fut = null;
+
+ switch (endOp) {
+ case 0:
+ tx.commit();
+
+ break;
+
+ case 1:
+ fut = tx.commitAsync();
+
+ break;
+
+ case 2:
+ tx.rollback();
+
+ break;
+
+ case 3:
+ fut = tx.rollbackAsync();
+
+ break;
+
+ case 4:
+ tx.close();
+
+ break;
+
+ default:
+ fail();
+ }
+
+ if (fut != null)
+ fut.get();
+
+ checkNoTx(node);
+ }
+
+ /**
+ * @param node Node.
+ */
+ private void checkNoTx(Ignite node) {
+ IgniteTransactions txs = node.transactions();
+
+ assertNull(txs.tx());
+ assertNull(((IgniteKernal)node).context().cache().context().tm().tx());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
index 37003a7..86c0fa4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteOptimisticTxSuspendResumeTest.java
@@ -46,7 +46,6 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionState.ACTIVE;
import static org.apache.ignite.transactions.TransactionState.COMMITTED;
-import static org.apache.ignite.transactions.TransactionState.MARKED_ROLLBACK;
import static org.apache.ignite.transactions.TransactionState.ROLLED_BACK;
import static org.apache.ignite.transactions.TransactionState.SUSPENDED;
@@ -60,6 +59,7 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
/** Future timeout */
private static final int FUT_TIMEOUT = 5000;
+ /** */
private boolean client = false;
/**
@@ -442,7 +442,7 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
}
}, TransactionTimeoutException.class);
- assertEquals(MARKED_ROLLBACK, tx.state());
+ assertEquals(ROLLED_BACK, tx.state());
tx.close();
}
@@ -476,7 +476,7 @@ public class IgniteOptimisticTxSuspendResumeTest extends GridCommonAbstractTest
}
}, TransactionTimeoutException.class);
- assertEquals(MARKED_ROLLBACK, tx.state());
+ assertEquals(ROLLED_BACK, tx.state());
tx.close();
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedMultiNodeLongTxTimeout2FullApiTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedMultiNodeLongTxTimeout2FullApiTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedMultiNodeLongTxTimeout2FullApiTest.java
new file mode 100644
index 0000000..aef63d0
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/near/CachePartitionedMultiNodeLongTxTimeout2FullApiTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class CachePartitionedMultiNodeLongTxTimeout2FullApiTest extends GridCachePartitionedMultiNodeFullApiSelfTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ cfg.getTransactionConfiguration().setDefaultTxTimeout(Long.MAX_VALUE / 2);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNearCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNearCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNearCacheTest.java
new file mode 100644
index 0000000..7c1a6dd
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNearCacheTest.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+/**
+ *
+ */
+public class TxRollbackOnTimeoutNearCacheTest extends TxRollbackOnTimeoutTest {
+ /** {@inheritDoc} */
+ @Override protected boolean nearCacheEnabled() {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java
new file mode 100644
index 0000000..5123329
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutNoDeadlockDetectionTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_TX_DEADLOCK_DETECTION_MAX_ITERS;
+
+/**
+ * Tests an ability to eagerly rollback timed out transactions.
+ */
+public class TxRollbackOnTimeoutNoDeadlockDetectionTest extends TxRollbackOnTimeoutTest {
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ System.setProperty(IGNITE_TX_DEADLOCK_DETECTION_MAX_ITERS, "0");
+
+ super.beforeTestsStarted();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ super.afterTestsStopped();
+
+ System.clearProperty(IGNITE_TX_DEADLOCK_DETECTION_MAX_ITERS);
+ }
+
+ /** */
+ @Override protected void validateDeadlockException(Exception e) {
+ assertEquals("TimeoutException is expected",
+ TransactionTimeoutException.class, e.getCause().getClass());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
new file mode 100644
index 0000000..e1c6c10
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/TxRollbackOnTimeoutTest.java
@@ -0,0 +1,655 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.transactions;
+
+import java.util.Collection;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ThreadLocalRandom;
+import javax.cache.CacheException;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.configuration.CacheConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.configuration.NearCacheConfiguration;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.G;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionDeadlockException;
+import org.apache.ignite.transactions.TransactionIsolation;
+import org.apache.ignite.transactions.TransactionTimeoutException;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL;
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * Tests an ability to eagerly rollback timed out transactions.
+ */
+public class TxRollbackOnTimeoutTest extends GridCommonAbstractTest {
+ /** */
+ private static final long TX_MIN_TIMEOUT = 1;
+
+ /** */
+ private static final String CACHE_NAME = "test";
+
+ /** IP finder. */
+ private static final TcpDiscoveryVmIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static final int GRID_CNT = 3;
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName);
+
+ ((TcpDiscoverySpi)cfg.getDiscoverySpi()).setIpFinder(IP_FINDER);
+
+ boolean client = "client".equals(igniteInstanceName);
+
+ cfg.setClientMode(client);
+
+ if (!client) {
+ CacheConfiguration ccfg = new CacheConfiguration(CACHE_NAME);
+
+ if (nearCacheEnabled())
+ ccfg.setNearConfiguration(new NearCacheConfiguration());
+
+ ccfg.setAtomicityMode(TRANSACTIONAL);
+ ccfg.setBackups(2);
+ ccfg.setWriteSynchronizationMode(FULL_SYNC);
+
+ cfg.setCacheConfiguration(ccfg);
+ }
+
+ return cfg;
+ }
+
+ /**
+ * @return Near cache flag.
+ */
+ protected boolean nearCacheEnabled() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ super.beforeTest();
+
+ startGridsMultiThreaded(GRID_CNT);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ super.afterTest();
+
+ stopAllGrids();
+ }
+
+ /**
+ * @throws Exception If f nodeailed.
+ * @return Started client.
+ */
+ private Ignite startClient() throws Exception {
+ Ignite client = startGrid("client");
+
+ assertTrue(client.configuration().isClientMode());
+
+ if (nearCacheEnabled())
+ client.createNearCache(CACHE_NAME, new NearCacheConfiguration<>());
+ else
+ assertNotNull(client.cache(CACHE_NAME));
+
+ return client;
+ }
+
+ /**
+ * @param e Exception.
+ */
+ protected void validateDeadlockException(Exception e) {
+ assertEquals("Deadlock report is expected",
+ TransactionDeadlockException.class, e.getCause().getCause().getClass());
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testLockAndConcurrentTimeout() throws Exception {
+ startClient();
+
+ for (Ignite node : G.allGrids()) {
+ log.info("Test with node: " + node.name());
+
+ lock(node, false);
+
+ lock(node, false);
+
+ lock(node, true);
+ }
+ }
+
+ /**
+ * @param node Node.
+ * @param retry {@code True}
+ * @throws Exception If failed.
+ */
+ private void lock(final Ignite node, final boolean retry) throws Exception {
+ final IgniteCache<Object, Object> cache = node.cache(CACHE_NAME);
+
+ final int KEYS_PER_THREAD = 10_000;
+
+ GridTestUtils.runMultiThreaded(new IgniteInClosure<Integer>() {
+ @Override public void apply(Integer idx) {
+ int start = idx * KEYS_PER_THREAD;
+ int end = start + KEYS_PER_THREAD;
+
+ int locked = 0;
+
+ try {
+ try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 500, 0)) {
+ for (int i = start; i < end; i++) {
+ cache.get(i);
+
+ locked++;
+ }
+
+ tx.commit();
+ }
+ }
+ catch (Exception e) {
+ info("Expected error: " + e);
+ }
+
+ info("Done, locked: " + locked);
+
+ if (retry) {
+ try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 10 * 60_000, 0)) {
+ for (int i = start; i < end; i++)
+ cache.get(i);
+
+ cache.put(start, 0);
+
+ tx.commit();
+ }
+ }
+ }
+ }, Math.min(4, Runtime.getRuntime().availableProcessors()), "tx-thread");
+ }
+
+ /**
+ * Tests if timeout on first tx unblocks second tx waiting for the locked key.
+ *
+ * @throws Exception If failed.
+ */
+ public void testWaitingTxUnblockedOnTimeout() throws Exception {
+ waitingTxUnblockedOnTimeout(grid(0), grid(0));
+
+ waitingTxUnblockedOnTimeout(grid(0), grid(1));
+
+ Ignite client = startClient();
+
+ waitingTxUnblockedOnTimeout(grid(0), client);
+
+ waitingTxUnblockedOnTimeout(grid(1), client);
+
+ waitingTxUnblockedOnTimeout(client, grid(0));
+
+ waitingTxUnblockedOnTimeout(client, grid(1));
+
+ waitingTxUnblockedOnTimeout(client, client);
+ }
+
+ /**
+ * Tests if timeout on first tx unblocks second tx waiting for the locked key.
+ *
+ * @throws Exception If failed.
+ */
+ public void testWaitingTxUnblockedOnThreadDeath() throws Exception {
+ waitingTxUnblockedOnThreadDeath(grid(0), grid(0));
+
+ waitingTxUnblockedOnThreadDeath(grid(0), grid(1));
+
+ Ignite client = startClient();
+
+ waitingTxUnblockedOnThreadDeath(grid(0), client);
+
+ waitingTxUnblockedOnThreadDeath(grid(1), client);
+
+ waitingTxUnblockedOnThreadDeath(client, grid(0));
+
+ waitingTxUnblockedOnThreadDeath(client, grid(1));
+
+ waitingTxUnblockedOnThreadDeath(client, client);
+ }
+
+ /**
+ * Tests if deadlock is resolved on timeout with correct message.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDeadlockUnblockedOnTimeout() throws Exception {
+ deadlockUnblockedOnTimeout(ignite(0), ignite(1));
+
+ deadlockUnblockedOnTimeout(ignite(0), ignite(0));
+
+ Ignite client = startClient();
+
+ deadlockUnblockedOnTimeout(ignite(0), client);
+
+ deadlockUnblockedOnTimeout(client, ignite(0));
+ }
+
+ /**
+ * Tests if deadlock is resolved on timeout with correct message.
+ *
+ * @param node1 First node.
+ * @param node2 Second node.
+ * @throws Exception If failed.
+ */
+ private void deadlockUnblockedOnTimeout(final Ignite node1, final Ignite node2) throws Exception {
+ info("Start test [node1=" + node1.name() + ", node2=" + node2.name() + ']');
+
+ final CountDownLatch l = new CountDownLatch(2);
+
+ IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try {
+ try (Transaction tx = node1.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 5000, 2)) {
+ node1.cache(CACHE_NAME).put(1, 10);
+
+ l.countDown();
+
+ U.awaitQuiet(l);
+
+ node1.cache(CACHE_NAME).put(2, 20);
+
+ tx.commit();
+
+ fail();
+ }
+ }
+ catch (CacheException e) {
+ // No-op.
+ validateDeadlockException(e);
+ }
+ }
+ }, "First");
+
+ IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try (Transaction tx = node2.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 2)) {
+ node2.cache(CACHE_NAME).put(2, 2);
+
+ l.countDown();
+
+ U.awaitQuiet(l);
+
+ node2.cache(CACHE_NAME).put(1, 1);
+
+ tx.commit();
+ }
+ }
+ }, "Second");
+
+ fut1.get();
+ fut2.get();
+
+ assertTrue("Expecting committed key 2", node1.cache(CACHE_NAME).get(2) != null);
+ assertTrue("Expecting committed key 1", node1.cache(CACHE_NAME).get(1) != null);
+
+ node1.cache(CACHE_NAME).removeAll(F.asSet(1, 2));
+ }
+
+ /**
+ * Tests timeout object cleanup on tx commit.
+ *
+ * @throws Exception If failed.
+ */
+ public void testTimeoutRemoval() throws Exception {
+ IgniteEx client = (IgniteEx)startClient();
+
+ final long TX_TIMEOUT = 250;
+
+ final int modesCnt = 5;
+
+ for (int i = 0; i < modesCnt; i++)
+ testTimeoutRemoval0(grid(0), i, TX_TIMEOUT);
+
+ for (int i = 0; i < modesCnt; i++)
+ testTimeoutRemoval0(client, i, TX_TIMEOUT);
+
+ for (int i = 0; i < modesCnt; i++)
+ testTimeoutRemoval0(grid(0), i, TX_MIN_TIMEOUT);
+
+ for (int i = 0; i < modesCnt; i++)
+ testTimeoutRemoval0(client, i, TX_MIN_TIMEOUT);
+
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ // Repeat with more iterations to make sure everything is cleared.
+ for (int i = 0; i < 500; i++)
+ testTimeoutRemoval0(client, rnd.nextInt(modesCnt), TX_MIN_TIMEOUT);
+ }
+
+ /**
+ * Tests timeouts in all tx configurations.
+ *
+ * @throws Exception If failed.
+ */
+ public void testSimple() throws Exception {
+ for (TransactionConcurrency concurrency : TransactionConcurrency.values())
+ for (TransactionIsolation isolation : TransactionIsolation.values()) {
+ for (int op = 0; op < 4; op++)
+ testSimple0(concurrency, isolation, op);
+ }
+ }
+
+ /**
+ * @param concurrency Concurrency.
+ * @param isolation Isolation.
+ * @param op Operation to test.
+ * @throws Exception If failed.
+ */
+ private void testSimple0(TransactionConcurrency concurrency, TransactionIsolation isolation, int op) throws Exception {
+ Ignite near = grid(0);
+
+ final int key = 1, val = 1;
+
+ final long TX_TIMEOUT = 250;
+
+ IgniteCache<Object, Object> cache = near.cache(CACHE_NAME);
+
+ try (Transaction tx = near.transactions().txStart(concurrency, isolation, TX_TIMEOUT, 1)) {
+ cache.put(key, val);
+
+ U.sleep(TX_TIMEOUT * 2);
+
+ try {
+ switch (op) {
+ case 0:
+ cache.put(key + 1, val);
+
+ break;
+
+ case 1:
+ cache.remove(key + 1);
+
+ break;
+
+ case 2:
+ cache.get(key + 1);
+
+ break;
+
+ case 3:
+ tx.commit();
+
+ break;
+
+ default:
+ fail();
+ }
+
+ fail("Tx must timeout");
+ }
+ catch (CacheException | IgniteException e) {
+ assertTrue("Expected exception: " + e, X.hasCause(e, TransactionTimeoutException.class));
+ }
+ }
+
+ assertFalse("Must be removed by rollback on timeout", near.cache(CACHE_NAME).containsKey(key));
+ assertFalse("Must be removed by rollback on timeout", near.cache(CACHE_NAME).containsKey(key + 1));
+
+ assertNull(near.transactions().tx());
+ }
+
+ /**
+ * @param near Node.
+ * @param mode Test mode.
+ *
+ * @param timeout Tx timeout.
+ * @throws Exception If failed.
+ */
+ private void testTimeoutRemoval0(IgniteEx near, int mode, long timeout) throws Exception {
+ Throwable saved = null;
+
+ try (Transaction tx = near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 1)) {
+ near.cache(CACHE_NAME).put(1, 1);
+
+ switch (mode) {
+ case 0:
+ tx.commit();
+ break;
+
+ case 1:
+ tx.commitAsync().get();
+ break;
+
+ case 2:
+ tx.rollback();
+ break;
+
+ case 3:
+ tx.rollbackAsync().get();
+ break;
+
+ case 4:
+ break;
+
+ default:
+ fail();
+ }
+ }
+ catch (Throwable t) {
+ saved = t;
+ }
+
+ Collection set = U.field(near.context().cache().context().time(), "timeoutObjs");
+
+ for (Object obj : set) {
+ if (obj.getClass().isAssignableFrom(GridNearTxLocal.class)) {
+ log.error("Last saved exception: " + saved, saved);
+
+ fail("Not removed [mode=" + mode + ", timeout=" + timeout + ", tx=" + obj +']');
+ }
+ }
+ }
+
+ /**
+ * @param near Node starting tx which is timed out.
+ * @param other Node starting second tx.
+ * @throws Exception If failed.
+ */
+ private void waitingTxUnblockedOnTimeout(final Ignite near, final Ignite other) throws Exception {
+ waitingTxUnblockedOnTimeout(near, other, 1000);
+
+ waitingTxUnblockedOnTimeout(near, other, 50);
+ }
+
+ /**
+ * @param near Node starting tx which is timed out.
+ * @param other Node starting second tx.
+ * @param timeout Timeout.
+ * @throws Exception If failed.
+ */
+ private void waitingTxUnblockedOnTimeout(final Ignite near, final Ignite other, final long timeout) throws Exception {
+ info("Start test [node1=" + near.name() + ", node2=" + other.name() + ']');
+
+ final CountDownLatch blocked = new CountDownLatch(1);
+
+ final CountDownLatch unblocked = new CountDownLatch(1);
+
+ final int recordsCnt = 5;
+
+ IgniteInternalFuture<?> fut1 = GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ try (Transaction tx = near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, timeout, 0)) {
+ try {
+ for (int i = 0; i < recordsCnt; i++)
+ near.cache(CACHE_NAME).put(i, i);
+
+ info("Locked all keys.");
+ }
+ catch (CacheException e) {
+ info("Failed to lock keys: " + e);
+ }
+ finally {
+ blocked.countDown();
+ }
+
+ // Will be unblocked after tx timeout occurs.
+ U.awaitQuiet(unblocked);
+
+ try {
+ near.cache(CACHE_NAME).put(0, 0);
+
+ fail();
+ }
+ catch (CacheException e) {
+ log.info("Expecting error: " + e.getMessage());
+ }
+
+ try {
+ tx.commit();
+
+ fail();
+ }
+ catch (IgniteException e) {
+ log.info("Expecting error: " + e.getMessage());
+ }
+ }
+
+ // Check thread is able to start new tx.
+ try (Transaction tx = near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 60_000, 0)) {
+ for (int i = 0; i < recordsCnt; i++)
+ near.cache(CACHE_NAME).put(i, i);
+
+ tx.commit();
+ }
+ }
+ }, "First");
+
+ IgniteInternalFuture<?> fut2 = GridTestUtils.runAsync(new Runnable() {
+ @Override public void run() {
+ U.awaitQuiet(blocked);
+
+ try (Transaction tx = other.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, 1)) {
+ for (int i = 0; i < recordsCnt; i++)
+ other.cache(CACHE_NAME).put(i, i);
+
+ // Will wait until timeout on first tx will unblock put.
+ tx.commit();
+ }
+ }
+ }, "Second");
+
+ fut2.get();
+
+ unblocked.countDown();
+
+ fut1.get();
+ }
+
+ /**
+ * @param near Node starting tx which is timed out.
+ * @param other Node starting second tx.
+ * @throws Exception If failed.
+ */
+ private void waitingTxUnblockedOnThreadDeath(final Ignite near, final Ignite other) throws Exception {
+ waitingTxUnblockedOnThreadDeath0(near, other, 10, 1000); // Try provoke timeout after all keys are locked.
+
+ waitingTxUnblockedOnThreadDeath0(near, other, 1000, 100); // Try provoke timeout while trying to lock keys.
+ }
+
+ /**
+ * @param near Node starting tx which is timed out.
+ * @param other Node starting second tx.
+ * @param recordsCnt Number of records to locks.
+ * @param timeout Transaction timeout.
+ * @throws Exception If failed.
+ */
+ private void waitingTxUnblockedOnThreadDeath0(final Ignite near,
+ final Ignite other,
+ final int recordsCnt,
+ final long timeout)
+ throws Exception
+ {
+ info("Start test [node1=" + near.name() + ", node2=" + other.name() + ']');
+
+ final CountDownLatch blocked = new CountDownLatch(1);
+
+ IgniteInternalFuture<?> fut1 = multithreadedAsync(new Runnable() {
+ @Override public void run() {
+ near.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, timeout, recordsCnt);
+
+ try {
+ for (int i = 0; i < recordsCnt; i++)
+ near.cache(CACHE_NAME).put(i, i);
+
+ log.info("Locked all records.");
+ }
+ catch (Exception e) {
+ log.info("Failed to locked all records: " + e);
+ }
+ finally {
+ blocked.countDown();
+ }
+
+ throw new IgniteException("Failure");
+ }
+ }, 1, "First");
+
+ IgniteInternalFuture<?> fut2 = multithreadedAsync(new Runnable() {
+ @Override public void run() {
+ U.awaitQuiet(blocked);
+
+ try (Transaction tx = other.transactions().txStart(PESSIMISTIC, REPEATABLE_READ, 0, recordsCnt)) {
+ for (int i = 0; i < recordsCnt; i++)
+ other.cache(CACHE_NAME).put(i, i);
+
+ // Will wait until timeout on first tx will unblock put.
+ tx.commit();
+ }
+ }
+ }, 1, "Second");
+
+ try {
+ fut1.get();
+
+ fail();
+ }
+ catch (IgniteCheckedException e) {
+ // No-op.
+ }
+
+ fut2.get();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
index 164ff6a..b380ebc 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheFullApiSelfTestSuite.java
@@ -31,6 +31,7 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePart
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledMultiNodeWithGroupFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledOnheapFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridCachePartitionedNearDisabledOnheapMultiNodeFullApiSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedMultiNodeLongTxTimeout2FullApiTest;
import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedMultiNodeLongTxTimeoutFullApiTest;
import org.apache.ignite.internal.processors.cache.distributed.near.CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridCacheAtomicClientOnlyMultiNodeFullApiSelfTest;
@@ -123,6 +124,7 @@ public class IgniteCacheFullApiSelfTestSuite extends TestSuite {
suite.addTestSuite(GridCacheAtomicMultiNodeP2PDisabledFullApiSelfTest.class);
suite.addTestSuite(GridCacheAtomicNearEnabledMultiNodeFullApiSelfTest.class);
suite.addTestSuite(CachePartitionedMultiNodeLongTxTimeoutFullApiTest.class);
+ suite.addTestSuite(CachePartitionedMultiNodeLongTxTimeout2FullApiTest.class);
suite.addTestSuite(CachePartitionedNearEnabledMultiNodeLongTxTimeoutFullApiTest.class);
suite.addTestSuite(GridCachePartitionedNearDisabledMultiNodeFullApiSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
index a0fb19b..7c71381 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheTestSuite6.java
@@ -21,9 +21,14 @@ import junit.framework.TestSuite;
import org.apache.ignite.internal.processors.cache.distributed.CacheExchangeMergeTest;
import org.apache.ignite.internal.processors.cache.distributed.CachePartitionStateTest;
import org.apache.ignite.internal.processors.cache.distributed.GridCachePartitionEvictionDuringReadThroughSelfTest;
+import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheThreadLocalTxTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeMultiServerTest;
import org.apache.ignite.internal.processors.cache.distributed.IgniteOptimisticTxSuspendResumeTest;
import org.apache.ignite.internal.processors.cache.distributed.IgnitePessimisticTxSuspendResumeTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNearCacheTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutNoDeadlockDetectionTest;
+import org.apache.ignite.internal.processors.cache.transactions.TxRollbackOnTimeoutTest;
+
/**
* Test suite.
*/
@@ -44,6 +49,11 @@ public class IgniteCacheTestSuite6 extends TestSuite {
suite.addTestSuite(CacheExchangeMergeTest.class);
+ suite.addTestSuite(TxRollbackOnTimeoutTest.class);
+ suite.addTestSuite(TxRollbackOnTimeoutNoDeadlockDetectionTest.class);
+ suite.addTestSuite(TxRollbackOnTimeoutNearCacheTest.class);
+ suite.addTestSuite(IgniteCacheThreadLocalTxTest.class);
+
return suite;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5af30cf1/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTxConfigCacheTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTxConfigCacheTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTxConfigCacheTest.java
index e85baed..f404732 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTxConfigCacheTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/impl/HadoopTxConfigCacheTest.java
@@ -25,11 +25,11 @@ import org.apache.ignite.internal.util.typedef.internal.CU;
/**
* Test checks whether hadoop system cache doesn't use user defined TX config.
*/
-public class HadoopTxConfigCacheTest extends IgniteTxConfigCacheSelfTest {
+public class HadoopTxConfigCacheTest extends IgniteTxConfigCacheSelfTest {
/**
* Success if system caches weren't timed out.
*
- * @throws Exception
+ * @throws Exception If failed.
*/
public void testSystemCacheTx() throws Exception {
final Ignite ignite = grid(0);
[14/35] ignite git commit: Updated github docs for .NET and C++
Posted by sb...@apache.org.
Updated github docs for .NET and C++
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/78dfb12b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/78dfb12b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/78dfb12b
Branch: refs/heads/ignite-3479
Commit: 78dfb12bce96e8ac0ed4eba0e0cd0e23adb81de6
Parents: f6f6c62
Author: Denis Magda <dm...@gridgain.com>
Authored: Thu Sep 21 13:25:36 2017 -0700
Committer: Denis Magda <dm...@gridgain.com>
Committed: Thu Sep 21 13:25:36 2017 -0700
----------------------------------------------------------------------
modules/platforms/cpp/README.md | 2 +-
modules/platforms/dotnet/README.md | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/78dfb12b/modules/platforms/cpp/README.md
----------------------------------------------------------------------
diff --git a/modules/platforms/cpp/README.md b/modules/platforms/cpp/README.md
index 8b89004..272f2d4 100644
--- a/modules/platforms/cpp/README.md
+++ b/modules/platforms/cpp/README.md
@@ -38,7 +38,7 @@ With the Ignite Persistence enabled, you no longer need to keep all the data and
<p align="center">
<a href="https://apacheignite.readme.io/docs/distributed-persistent-store">
- <img src="https://ignite.apache.org/images/native_persistence.png" width="400px"/>
+ <img src="https://ignite.apache.org/images/native_persistence.png?renew" width="400px"/>
</a>
</p>
http://git-wip-us.apache.org/repos/asf/ignite/blob/78dfb12b/modules/platforms/dotnet/README.md
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/README.md b/modules/platforms/dotnet/README.md
index d2d6191..4487c00 100644
--- a/modules/platforms/dotnet/README.md
+++ b/modules/platforms/dotnet/README.md
@@ -43,7 +43,7 @@ With the Ignite Persistence enabled, you no longer need to keep all the data and
<p align="center">
<a href="https://apacheignite.readme.io/docs/distributed-persistent-store">
- <img src="https://ignite.apache.org/images/native_persistence.png" width="400px"/>
+ <img src="https://ignite.apache.org/images/native_persistence.png?renew" width="400px"/>
</a>
</p>
[11/35] ignite git commit: Merge remote-tracking branch
'origin/master'
Posted by sb...@apache.org.
Merge remote-tracking branch 'origin/master'
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/271b32f6
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/271b32f6
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/271b32f6
Branch: refs/heads/ignite-3479
Commit: 271b32f649a746095a9391e28e48d4f3c50ea14c
Parents: 4b4acd3 db96b79
Author: devozerov <pp...@gmail.com>
Authored: Thu Sep 21 20:04:09 2017 +0300
Committer: devozerov <pp...@gmail.com>
Committed: Thu Sep 21 20:04:09 2017 +0300
----------------------------------------------------------------------
.../ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------