You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2018/10/01 05:55:11 UTC
[01/21] ignite git commit: IGNITE-8950 More informative file
validation check message - Fixes #4322.
Repository: ignite
Updated Branches:
refs/heads/ignite-gg-14206 b12d73802 -> 9ab8ebd7a
IGNITE-8950 More informative file validation check message - Fixes #4322.
Signed-off-by: Dmitriy Pavlov <dp...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/66acc566
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/66acc566
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/66acc566
Branch: refs/heads/ignite-gg-14206
Commit: 66acc566a939e1e7c051b73f1525c7f23dd7ea1d
Parents: b12d738
Author: Evgeny Stanilovskiy <es...@gridgain.com>
Authored: Fri Sep 28 10:04:45 2018 +0300
Committer: Dmitriy Pavlov <dp...@apache.org>
Committed: Fri Sep 28 10:04:45 2018 +0300
----------------------------------------------------------------------
.../cache/persistence/file/FilePageStore.java | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/66acc566/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
index 4bb7513..110807c 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/file/FilePageStore.java
@@ -194,29 +194,31 @@ public class FilePageStore implements PageStore {
long signature = hdr.getLong();
+ String prefix = "Failed to verify, file=" + cfgFile.getAbsolutePath() + "\" ";
+
if (SIGNATURE != signature)
- throw new IOException("Failed to verify store file (invalid file signature)" +
+ throw new IOException(prefix + "(invalid file signature)" +
" [expectedSignature=" + U.hexLong(SIGNATURE) +
", actualSignature=" + U.hexLong(signature) + ']');
int ver = hdr.getInt();
if (version() != ver)
- throw new IOException("Failed to verify store file (invalid file version)" +
+ throw new IOException(prefix + "(invalid file version)" +
" [expectedVersion=" + version() +
", fileVersion=" + ver + "]");
byte type = hdr.get();
if (this.type != type)
- throw new IOException("Failed to verify store file (invalid file type)" +
+ throw new IOException(prefix + "(invalid file type)" +
" [expectedFileType=" + this.type +
", actualFileType=" + type + "]");
int pageSize = hdr.getInt();
if (dbCfg.getPageSize() != pageSize)
- throw new IOException("Failed to verify store file (invalid page size)" +
+ throw new IOException(prefix + "(invalid page size)" +
" [expectedPageSize=" + dbCfg.getPageSize() +
", filePageSize=" + pageSize + "]");
@@ -226,7 +228,7 @@ public class FilePageStore implements PageStore {
fileSize = pageSize + headerSize();
if ((fileSize - headerSize()) % pageSize != 0)
- throw new IOException("Failed to verify store file (invalid file size)" +
+ throw new IOException(prefix + "(invalid file size)" +
" [fileSize=" + U.hexLong(fileSize) +
", pageSize=" + U.hexLong(pageSize) + ']');
[20/21] ignite git commit: IGNITE-9612 Fixed checkpoint listener
logic - Fixes #4864.
Posted by sb...@apache.org.
IGNITE-9612 Fixed checkpoint listener logic - Fixes #4864.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2f3b567f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2f3b567f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2f3b567f
Branch: refs/heads/ignite-gg-14206
Commit: 2f3b567f61c2a9a1080ee7fda57f3a18231b1dae
Parents: 69adfd5
Author: Aleksei Scherbakov <al...@gmail.com>
Authored: Fri Sep 28 18:50:05 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 28 18:50:05 2018 +0300
----------------------------------------------------------------------
.../persistence/GridCacheOffheapManager.java | 29 ++++++++++----------
1 file changed, 15 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/2f3b567f/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index 16c30c9..d704abd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -170,12 +170,24 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
Executor execSvc = ctx.executor();
+ if (ctx.nextSnapshot() && ctx.needToSnapshot(grp.cacheOrGroupName())) {
+ if (execSvc == null)
+ updateSnapshotTag(ctx);
+ else {
+ execSvc.execute(() -> {
+ try {
+ updateSnapshotTag(ctx);
+ }
+ catch (IgniteCheckedException e) {
+ throw new IgniteException(e);
+ }
+ });
+ }
+ }
+
if (execSvc == null) {
reuseList.saveMetadata();
- if (ctx.nextSnapshot())
- updateSnapshotTag(ctx);
-
for (CacheDataStore store : partDataStores.values())
saveStoreMetadata(store, ctx, false);
}
@@ -189,17 +201,6 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
}
});
- if (ctx.nextSnapshot()) {
- execSvc.execute(() -> {
- try {
- updateSnapshotTag(ctx);
- }
- catch (IgniteCheckedException e) {
- throw new IgniteException(e);
- }
- });
- }
-
for (CacheDataStore store : partDataStores.values())
execSvc.execute(() -> {
try {
[21/21] ignite git commit: IGNITE-9569 Web Console: Fixed broken unit
test.
Posted by sb...@apache.org.
IGNITE-9569 Web Console: Fixed broken unit test.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/9ab8ebd7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/9ab8ebd7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/9ab8ebd7
Branch: refs/heads/ignite-gg-14206
Commit: 9ab8ebd7adaa70548ba013f4d8c7de05ba48312f
Parents: 2f3b567
Author: Ilya Borisov <kl...@gmail.com>
Authored: Mon Oct 1 10:45:40 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Mon Oct 1 10:45:40 2018 +0700
----------------------------------------------------------------------
modules/web-console/frontend/test/unit/defaultName.filter.test.js | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/9ab8ebd7/modules/web-console/frontend/test/unit/defaultName.filter.test.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/test/unit/defaultName.filter.test.js b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
index 2eaf223..7ecbd6f 100644
--- a/modules/web-console/frontend/test/unit/defaultName.filter.test.js
+++ b/modules/web-console/frontend/test/unit/defaultName.filter.test.js
@@ -17,10 +17,9 @@
import defaultName from '../../app/filters/default-name.filter';
-import { suite, test } from 'mocha';
import { assert } from 'chai';
-const instance = defaultName[0]();
+const instance = defaultName();
suite('defaultName', () => {
test('defaultName filter', () => {
[10/21] ignite git commit: IGNITE-7764: MVCC: cache API support. This
closes #4725.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
index 83bb81c..af74996 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccTransactionsTest.java
@@ -20,8 +20,10 @@ package org.apache.ignite.internal.processors.cache.mvcc;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -64,7 +66,6 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetR
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishResponse;
-import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareResponse;
import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccAckRequestQueryCntr;
import org.apache.ignite.internal.processors.cache.mvcc.msg.MvccAckRequestTx;
@@ -86,9 +87,7 @@ import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.testframework.GridTestUtils;
import org.apache.ignite.transactions.Transaction;
-import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
-import org.apache.ignite.transactions.TransactionOptimisticException;
import org.jetbrains.annotations.Nullable;
import org.junit.Assert;
@@ -98,11 +97,8 @@ import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstract
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.PUT;
import static org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker.MVCC_TRACKER_ID_NA;
-import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
-import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
-import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
/**
* TODO IGNITE-6739: tests reload
@@ -117,40 +113,104 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
}
/**
- * @throws Exception If failed.
+ * @throws Exception if failed.
*/
- public void testPessimisticTx1() throws Exception {
- checkTx1(PESSIMISTIC, REPEATABLE_READ);
- }
+ public void testEmptyTx() throws Exception {
+ Ignite node = startGrids(2);
- /**
- * @throws Exception If failed.
- */
- public void testOptimisticSerializableTx1() throws Exception {
- checkTx1(OPTIMISTIC, SERIALIZABLE);
- }
+ IgniteCache cache = node.createCache(cacheConfiguration(PARTITIONED, FULL_SYNC, 1, DFLT_PARTITION_COUNT));
- /**
- * @throws Exception If failed.
- */
- public void testOptimisticRepeatableReadTx1() throws Exception {
- checkTx1(OPTIMISTIC, REPEATABLE_READ);
+ cache.putAll(Collections.emptyMap());
+
+ IgniteTransactions txs = node.transactions();
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ tx.commit();
+ }
+ finally {
+ stopAllGrids();
+ }
}
/**
- * @throws Exception If failed.
+ * @throws Exception if failed.
*/
- public void testOptimisticReadCommittedTx1() throws Exception {
- checkTx1(OPTIMISTIC, READ_COMMITTED);
+ public void testImplicitTxOps() throws Exception {
+ checkTxWithAllCaches(new CI1<IgniteCache<Integer, Integer>>() {
+ @Override public void apply(IgniteCache<Integer, Integer> cache) {
+ try {
+ List<Integer> keys = testKeys(cache);
+
+ for (Integer key : keys) {
+ log.info("Test key: " + key);
+
+ Integer val = cache.get(key);
+
+ assertNull(val);
+
+ assertFalse(cache.containsKey(key));
+
+ cache.put(key, -1);
+
+ val = (Integer)checkAndGet(true, cache, key, GET, SCAN);
+
+ assertEquals(Integer.valueOf(-1), val);
+
+ assertTrue(cache.containsKey(key));
+
+ cache.put(key, key);
+
+ val = (Integer)checkAndGet(true, cache, key, GET, SCAN);
+
+ assertEquals(key, val);
+
+ cache.remove(key);
+
+ val = cache.get(key);
+
+ assertNull(val);
+
+ val = (Integer)checkAndGet(false, cache, key, SCAN, GET);
+
+ assertNull(val);
+
+ assertTrue(cache.putIfAbsent(key, key));
+
+ val = (Integer)checkAndGet(true, cache, key, GET, SCAN);
+
+ assertEquals(key, val);
+
+ val = cache.getAndReplace(key, -1);
+
+ assertEquals(key, val);
+
+ val = (Integer)checkAndGet(true, cache, key, GET, SCAN);
+
+ assertEquals(Integer.valueOf(-1), val);
+
+ val = cache.getAndRemove(key);
+
+ assertEquals(Integer.valueOf(-1), val);
+
+ val = cache.get(key);
+
+ assertNull(val);
+
+ val = (Integer)checkAndGet(false, cache, key, SCAN, GET);
+
+ assertNull(val);
+ }
+ }
+ catch (Exception e) {
+ throw new IgniteException(e);
+ }
+ }
+ });
}
/**
- * @param concurrency Transaction concurrency.
- * @param isolation Transaction isolation.
* @throws Exception If failed.
*/
- private void checkTx1(final TransactionConcurrency concurrency, final TransactionIsolation isolation)
- throws Exception {
+ public void testPessimisticTx1() throws Exception {
checkTxWithAllCaches(new CI1<IgniteCache<Integer, Integer>>() {
@Override public void apply(IgniteCache<Integer, Integer> cache) {
try {
@@ -161,7 +221,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
for (Integer key : keys) {
log.info("Test key: " + key);
- try (Transaction tx = txs.txStart(concurrency, isolation)) {
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
Integer val = cache.get(key);
assertNull(val);
@@ -191,23 +251,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testPessimisticTx2() throws Exception {
- checkTx2(PESSIMISTIC, REPEATABLE_READ);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testOptimisticSerializableTx2() throws Exception {
- checkTx2(OPTIMISTIC, SERIALIZABLE);
- }
-
- /**
- * @param concurrency Transaction concurrency.
- * @param isolation Transaction isolation.
- * @throws Exception If failed.
- */
- private void checkTx2(final TransactionConcurrency concurrency, final TransactionIsolation isolation)
- throws Exception {
checkTxWithAllCaches(new CI1<IgniteCache<Integer, Integer>>() {
@Override public void apply(IgniteCache<Integer, Integer> cache) {
try {
@@ -218,7 +261,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
for (Integer key : keys) {
log.info("Test key: " + key);
- try (Transaction tx = txs.txStart(concurrency, isolation)) {
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
cache.put(key, key);
cache.put(key + 1, key + 1);
@@ -394,7 +437,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
keys.add(rnd.nextInt());
if (tx) {
- try (Transaction tx = txs.txStart(OPTIMISTIC, SERIALIZABLE)) {
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
cache.getAll(keys);
if (rnd.nextBoolean())
@@ -417,8 +460,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testTxReadIsolationSimple() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-7764");
-
Ignite srv0 = startGrids(4);
client = true;
@@ -453,37 +494,23 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
@Override public Void call() throws Exception {
IgniteCache<Object, Object> cache = node.cache(DEFAULT_CACHE_NAME);
- try (Transaction tx = node.transactions().txStart(OPTIMISTIC, isolation)) {
+ try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
assertEquals(0, checkAndGet(false, cache, 0, SCAN, GET));
readStart.countDown();
assertTrue(readProceed.await(5, TimeUnit.SECONDS));
- if (isolation == READ_COMMITTED) {
- assertNull(checkAndGet(false, cache, 1, SCAN, GET));
-
- assertEquals(1, checkAndGet(false, cache, 2, SCAN, GET));
-
- Map<Object, Object> res = checkAndGetAll(false, cache, startVals.keySet(), SCAN, GET);
-
- assertEquals(startVals.size() / 2, res.size());
+ assertEquals(0, checkAndGet(true, cache, 1, GET, SCAN));
- for (Map.Entry<Object, Object> e : res.entrySet())
- assertEquals("Invalid value for key: " + e.getKey(), 1, e.getValue());
- }
- else {
- assertEquals(0, checkAndGet(true, cache, 1, GET, SCAN));
-
- assertEquals(0, checkAndGet(true, cache, 2, GET, SCAN));
+ assertEquals(0, checkAndGet(true, cache, 2, GET, SCAN));
- Map<Object, Object> res = checkAndGetAll(true, cache, startVals.keySet(), GET, SCAN);
+ Map<Object, Object> res = checkAndGetAll(true, cache, startVals.keySet(), GET, SCAN);
- assertEquals(startVals.size(), res.size());
+ assertEquals(startVals.size(), res.size());
- for (Map.Entry<Object, Object> e : res.entrySet())
- assertEquals("Invalid value for key: " + e.getKey(), 0, e.getValue());
- }
+ for (Map.Entry<Object, Object> e : res.entrySet())
+ assertEquals("Invalid value for key: " + e.getKey(), 0, e.getValue());
tx.rollback();
}
@@ -589,8 +616,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
}
/**
- * @throws Exception If failed.
* @param largeKeys {@code True} to use large keys (not fitting in single page).
+ * @throws Exception If failed.
*/
private void putRemoveSimple(boolean largeKeys) throws Exception {
Ignite node = startGrid(0);
@@ -771,6 +798,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testWaitPreviousTxAck() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
+
testSpi = true;
startGrid(0);
@@ -1032,11 +1061,10 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
@Override public Void call() throws Exception {
IgniteCache<Integer, Integer> cache = client.cache(srvCache.getName());
-
Map<Integer, Integer> vals;
if (inTx) {
- try (Transaction tx = client.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
vals = checkAndGetAll(false, cache, F.asSet(key1, key2), SCAN, GET);
tx.rollback();
@@ -1085,6 +1113,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testCleanupWaitsForGet2() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
/*
Simulate case when there are two active transactions modifying the same key
(it is possible if key lock is released but ack message is delayed), and at this moment
@@ -1430,8 +1459,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
putAllGetAll(RestartMode.RESTART_RND_SRV, 4, 2, 1, 64, null, SCAN, PUT);
}
-
-
/**
* @throws Exception If failed.
*/
@@ -1576,79 +1603,35 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testPessimisticTxGetAllReadsSnapshot_SingleNode_SinglePartition() throws Exception {
- txReadsSnapshot(1, 0, 0, 1, true, GET);
+ txReadsSnapshot(1, 0, 0, 1, GET);
}
/**
* @throws Exception If failed.
*/
public void testPessimisticTxGetAllReadsSnapshot_ClientServer() throws Exception {
- txReadsSnapshot(4, 2, 1, 64, true, GET);
+ txReadsSnapshot(4, 2, 1, 64, GET);
}
/**
* @throws Exception If failed.
*/
- public void testOptimisticTxGetAllReadsSnapshot_SingleNode() throws Exception {
- txReadsSnapshot(1, 0, 0, 64, false, GET);
+ public void testPessimisticTxScanReadsSnapshot_SingleNode_SinglePartition() throws Exception {
+ txReadsSnapshot(1, 0, 0, 1, SCAN);
}
/**
* @throws Exception If failed.
*/
- public void testOptimisticTxGetAllReadsSnapshot_SingleNode_SinglePartition() throws Exception {
- txReadsSnapshot(1, 0, 0, 1, false, GET);
+ public void testPessimisticTxScanReadsSnapshot_ClientServer() throws Exception {
+ txReadsSnapshot(4, 2, 1, 64, SCAN);
}
/**
- * @throws Exception If failed.
- */
- public void testOptimisticTxGetAllReadsSnapshot_ClientServer() throws Exception {
- txReadsSnapshot(4, 2, 1, 64, false, GET);
- }
-
-// TODO: IGNITE-7371
-// /**
-// * @throws Exception If failed.
-// */
-// public void testPessimisticTxScanReadsSnapshot_SingleNode_SinglePartition() throws Exception {
-// txReadsSnapshot(1, 0, 0, 1, true, SCAN);
-// }
-//
-// /**
-// * @throws Exception If failed.
-// */
-// public void testPessimisticTxScanReadsSnapshot_ClientServer() throws Exception {
-// txReadsSnapshot(4, 2, 1, 64, true, SCAN);
-// }
-//
-// /**
-// * @throws Exception If failed.
-// */
-// public void testOptimisticTxScanReadsSnapshot_SingleNode() throws Exception {
-// txReadsSnapshot(1, 0, 0, 64, false, SCAN);
-// }
-//
-// /**
-// * @throws Exception If failed.
-// */
-// public void testOptimisticTxScanReadsSnapshot_SingleNode_SinglePartition() throws Exception {
-// txReadsSnapshot(1, 0, 0, 1, false, SCAN);
-// }
-//
-// /**
-// * @throws Exception If failed.
-// */
-// public void testOptimisticTxScanReadsSnapshot_ClientServer() throws Exception {
-// txReadsSnapshot(4, 2, 1, 64, false, SCAN);
-// }
-
- /**
* @param srvs Number of server nodes.
* @param clients Number of client nodes.
* @param cacheBackups Number of cache backups.
* @param cacheParts Number of cache partitions.
- * @param pessimistic If {@code true} uses pessimistic tx, otherwise optimistic.
* @param readMode Read mode.
* @throws Exception If failed.
*/
@@ -1657,9 +1640,10 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
final int clients,
int cacheBackups,
int cacheParts,
- final boolean pessimistic,
ReadMode readMode
) throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
+
final int ACCOUNTS = 20;
final int ACCOUNT_START_VAL = 1000;
@@ -1668,18 +1652,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
final int readers = 4;
- final TransactionConcurrency concurrency;
- final TransactionIsolation isolation;
-
- if (pessimistic) {
- concurrency = PESSIMISTIC;
- isolation = REPEATABLE_READ;
- }
- else {
- concurrency = OPTIMISTIC;
- isolation = SERIALIZABLE;
- }
-
final IgniteInClosure<IgniteCache<Object, Object>> init = new IgniteInClosure<IgniteCache<Object, Object>>() {
@Override public void apply(IgniteCache<Object, Object> cache) {
final IgniteTransactions txs = cache.unwrap(Ignite.class).transactions();
@@ -1689,7 +1661,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
for (int i = 0; i < ACCOUNTS; i++)
accounts.put(i, new MvccTestAccount(ACCOUNT_START_VAL, 1));
- try (Transaction tx = txs.txStart(concurrency, isolation)) {
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
cache.putAll(accounts);
tx.commit();
@@ -1718,7 +1690,13 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
while (id1.equals(id2))
id2 = rnd.nextInt(ACCOUNTS);
- TreeSet<Integer> keys = new TreeSet<>();
+ if(id1 > id2) {
+ int tmp = id1;
+ id1 = id2;
+ id2 = tmp;
+ }
+
+ Set<Integer> keys = new HashSet<>();
keys.add(id1);
keys.add(id2);
@@ -1763,88 +1741,36 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
Map<Integer, MvccTestAccount> accounts = new HashMap<>();
- if (pessimistic) {
- try (Transaction tx = txs.txStart(concurrency, isolation)) {
- int remaining = ACCOUNTS;
-
- do {
- int readCnt = rnd.nextInt(remaining) + 1;
-
- Set<Integer> readKeys = new TreeSet<>();
-
- for (int i = 0; i < readCnt; i++)
- readKeys.add(accounts.size() + i);
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ int remaining = ACCOUNTS;
- Map<Integer, MvccTestAccount> readRes =
- checkAndGetAll(false, cache.cache, readKeys, readMode);
+ do {
+ int readCnt = rnd.nextInt(remaining) + 1;
- assertEquals(readCnt, readRes.size());
+ Set<Integer> readKeys = new TreeSet<>();
- accounts.putAll(readRes);
+ for (int i = 0; i < readCnt; i++)
+ readKeys.add(accounts.size() + i);
- remaining = ACCOUNTS - accounts.size();
- }
- while (remaining > 0);
+ Map<Integer, MvccTestAccount> readRes =
+ checkAndGetAll(false, cache.cache, readKeys, readMode);
- validateSum(accounts);
+ assertEquals(readCnt, readRes.size());
- tx.commit();
+ accounts.putAll(readRes);
- cnt++;
+ remaining = ACCOUNTS - accounts.size();
}
- finally {
- cache.readUnlock();
- }
- }
- else {
- try (Transaction tx = txs.txStart(concurrency, isolation)) {
- int remaining = ACCOUNTS;
-
- do {
- int readCnt = rnd.nextInt(remaining) + 1;
-
- if (rnd.nextInt(3) == 0) {
- for (int i = 0; i < readCnt; i++) {
- Integer key = rnd.nextInt(ACCOUNTS);
-
- MvccTestAccount account =
- (MvccTestAccount)checkAndGet(false, cache.cache, key, readMode);
-
- assertNotNull(account);
-
- accounts.put(key, account);
- }
- }
- else {
- Set<Integer> readKeys = new LinkedHashSet<>();
-
- for (int i = 0; i < readCnt; i++)
- readKeys.add(rnd.nextInt(ACCOUNTS));
-
- Map<Integer, MvccTestAccount> readRes =
- checkAndGetAll(false, cache.cache, readKeys, readMode);
+ while (remaining > 0);
- assertEquals(readKeys.size(), readRes.size());
-
- accounts.putAll(readRes);
- }
+ validateSum(accounts);
- remaining = ACCOUNTS - accounts.size();
- }
- while (remaining > 0);
-
- validateSum(accounts);
-
- cnt++;
+ tx.commit();
- tx.commit();
- }
- catch (TransactionOptimisticException ignore) {
- // No-op.
- }
- finally {
- cache.readUnlock();
- }
+ cnt++;
+ }
+ finally {
+ cache.readUnlock();
}
}
@@ -1954,9 +1880,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
int cacheBackups,
int cacheParts,
ReadMode readMode
- )
- throws Exception
- {
+ ) throws Exception {
final int writers = 4;
final int readers = 4;
@@ -2103,50 +2027,26 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
ThreadLocalRandom rnd = ThreadLocalRandom.current();
Map<Integer, Integer> map = new TreeMap<>();
+ Set<Integer> keys = new LinkedHashSet();
int cnt = 0;
while (!stop.get()) {
- int keys = rnd.nextInt(32) + 1;
+ int keysCnt = rnd.nextInt(32) + 1;
- while (map.size() < keys)
- map.put(rnd.nextInt(KEYS), cnt);
+ while (keys.size() < keysCnt) {
+ int key = rnd.nextInt(KEYS);
+
+ if(keys.add(key))
+ map.put(key, cnt);
+ }
TestCache<Integer, Integer> cache = randomCache(caches, rnd);
try {
IgniteTransactions txs = cache.cache.unwrap(Ignite.class).transactions();
- TransactionConcurrency concurrency;
- TransactionIsolation isolation;
-
- switch (rnd.nextInt(3)) {
- case 0: {
- concurrency = PESSIMISTIC;
- isolation = REPEATABLE_READ;
-
- break;
- }
- case 1: {
- concurrency = OPTIMISTIC;
- isolation = REPEATABLE_READ;
-
- break;
- }
- case 2: {
- concurrency = OPTIMISTIC;
- isolation = SERIALIZABLE;
-
- break;
- }
- default: {
- fail();
-
- return;
- }
- }
-
- try (Transaction tx = txs.txStart(concurrency, isolation)) {
+ try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
if (rnd.nextBoolean()) {
Map<Integer, Integer> res = checkAndGetAll(false, cache.cache, map.keySet(),
rnd.nextBoolean() ? GET : SCAN);
@@ -2158,18 +2058,16 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
tx.commit();
}
- catch (TransactionOptimisticException e) {
- assertEquals(SERIALIZABLE, isolation);
- }
catch (Exception e) {
Assert.assertTrue("Unexpected error: " + e, X.hasCause(e, ClusterTopologyException.class));
}
}
finally {
cache.readUnlock();
- }
- map.clear();
+ keys.clear();
+ map.clear();
+ }
cnt++;
}
@@ -2282,7 +2180,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
Ignite srv0 = startGrid(0);
- IgniteCache<Integer, Integer> cache = (IgniteCache)srv0.createCache(
+ IgniteCache<Integer, Integer> cache = (IgniteCache)srv0.createCache(
cacheConfiguration(PARTITIONED, FULL_SYNC, 0, DFLT_PARTITION_COUNT));
Map<Integer, Integer> map;
@@ -2413,32 +2311,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testTxPrepareFailureSimplePessimisticTx() throws Exception {
- txPrepareFailureSimple(PESSIMISTIC, REPEATABLE_READ);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTxPrepareFailureSimpleSerializableTx() throws Exception {
- txPrepareFailureSimple(OPTIMISTIC, SERIALIZABLE);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTxPrepareFailureSimpleOptimisticTx() throws Exception {
- txPrepareFailureSimple(OPTIMISTIC, REPEATABLE_READ);
- }
-
- /**
- * @param concurrency Transaction concurrency.
- * @param isolation Transaction isolation.
- * @throws Exception If failed.
- */
- private void txPrepareFailureSimple(
- final TransactionConcurrency concurrency,
- final TransactionIsolation isolation
- ) throws Exception {
testSpi = true;
startGrids(3);
@@ -2460,7 +2332,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() {
@Override public Object call() throws Exception {
try {
- try (Transaction tx = client.transactions().txStart(concurrency, isolation)) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
cache.put(key1, 1);
cache.put(key2, 2);
@@ -2477,7 +2349,16 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
}
}, "tx-thread");
- srv1Spi.waitForBlocked();
+ GridTestUtils.waitForCondition(
+ new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return srv1Spi.hasBlockedMessages() || fut.isDone() && fut.error() != null;
+ }
+ }, 10_000
+ );
+
+ if (fut.isDone())
+ fut.get(); // Just to fail with future error.
assertFalse(fut.isDone());
@@ -2488,7 +2369,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
assertNull(cache.get(key1));
assertNull(cache.get(key2));
- try (Transaction tx = client.transactions().txStart(concurrency, isolation)) {
+ try (Transaction tx = client.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
cache.put(key1, 1);
cache.put(key2, 2);
@@ -2502,64 +2383,9 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
/**
* @throws Exception If failed.
*/
- public void testSerializableTxRemap() throws Exception {
- testSpi = true;
-
- startGrids(2);
-
- client = true;
-
- final Ignite client = startGrid(2);
-
- final IgniteCache cache = client.createCache(
- cacheConfiguration(PARTITIONED, FULL_SYNC, 0, DFLT_PARTITION_COUNT));
-
- final Map<Object, Object> vals = new HashMap<>();
-
- for (int i = 0; i < 100; i++)
- vals.put(i, i);
-
- TestRecordingCommunicationSpi clientSpi = TestRecordingCommunicationSpi.spi(ignite(2));
-
- clientSpi.blockMessages(new IgniteBiPredicate<ClusterNode, Message>() {
- @Override public boolean apply(ClusterNode node, Message msg) {
- return msg instanceof GridNearTxPrepareRequest;
- }
- });
-
- IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable() {
- @Override public Object call() throws Exception {
- try (Transaction tx = client.transactions().txStart(OPTIMISTIC, SERIALIZABLE)) {
- cache.putAll(vals);
-
- tx.commit();
- }
-
- return null;
- }
- }, "tx-thread");
-
- clientSpi.waitForBlocked(2);
-
- this.client = false;
-
- startGrid(3);
-
- assertFalse(fut.isDone());
-
- clientSpi.stopBlock();
-
- fut.get();
-
- for (Ignite node : G.allGrids())
- checkValues(vals, node.cache(cache.getName()));
- }
-
-
- /**
- * @throws Exception If failed.
- */
public void testMvccCoordinatorChangeSimple() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9722");
+
Ignite srv0 = startGrid(0);
final List<String> cacheNames = new ArrayList<>();
@@ -2624,19 +2450,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
for (int i = 0; i < 10; i++)
vals.put(i, val);
- TransactionConcurrency concurrency;
- TransactionIsolation isolation;
-
- if (ThreadLocalRandom.current().nextBoolean()) {
- concurrency = PESSIMISTIC;
- isolation = REPEATABLE_READ;
- }
- else {
- concurrency = OPTIMISTIC;
- isolation = SERIALIZABLE;
- }
-
- try (Transaction tx = putNode.transactions().txStart(concurrency, isolation)) {
+ try (Transaction tx = putNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
for (String cacheName : cacheNames)
putNode.cache(cacheName).putAll(vals);
@@ -2906,8 +2720,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws Exception If failed.
*/
public void testUpdate_N_Objects_ClientServer_Backups1_Scan() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-7764");
-
int[] nValues = {3, 5, 10};
for (int n : nValues) {
@@ -2924,7 +2736,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
doImplicitPartsScanTest(1, 0, 0, 1, 10_000);
}
-
/**
* @throws Exception If failed.
*/
@@ -2967,6 +2778,8 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
int cacheBackups,
int cacheParts,
long time) throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
+
final int KEYS_PER_PART = 20;
final int writers = 4;
@@ -3033,6 +2846,12 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
while (k1.equals(k2))
k2 = partKeys.get(rnd.nextInt(KEYS_PER_PART));
+ if(k1 > k2) {
+ int tmp = k1;
+ k1 = k2;
+ k2 = tmp;
+ }
+
TreeSet<Integer> keys = new TreeSet<>();
keys.add(k1);
@@ -3074,7 +2893,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
ScanQuery<Integer, MvccTestAccount> qry = new ScanQuery<>(part);
- List<Cache.Entry<Integer, MvccTestAccount>> res = cache.cache.query(qry).getAll();
+ List<Cache.Entry<Integer, MvccTestAccount>> res = cache.cache.query(qry).getAll();
int sum = 0;
@@ -3100,7 +2919,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
try {
ScanQuery<Integer, MvccTestAccount> qry = new ScanQuery<>();
- List<Cache.Entry<Integer, MvccTestAccount>> res = cache.cache.query(qry).getAll();
+ List<Cache.Entry<Integer, MvccTestAccount>> res = cache.cache.query(qry).getAll();
int sum = 0;
@@ -3140,8 +2959,6 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @throws IgniteCheckedException If failed.
*/
public void testSize() throws Exception {
- fail("https://issues.apache.org/jira/browse/IGNITE-9451");
-
Ignite node = startGrid(0);
IgniteCache cache = node.createCache(cacheConfiguration(PARTITIONED, FULL_SYNC, 0, 1));
@@ -3150,6 +2967,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
final int KEYS = 10;
+ // Initial put.
for (int i = 0; i < KEYS; i++) {
final Integer key = i;
@@ -3162,6 +2980,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
assertEquals(i + 1, cache.size());
}
+ // Update.
for (int i = 0; i < KEYS; i++) {
final Integer key = i;
@@ -3176,6 +2995,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
int size = KEYS;
+ // Remove.
for (int i = 0; i < KEYS; i++) {
if (i % 2 == 0) {
final Integer key = i;
@@ -3207,6 +3027,48 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
}
}
+ // Check rollback create.
+ for (int i = 0; i < KEYS; i++) {
+ if (i % 2 == 0) {
+ final Integer key = i;
+
+ try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(key, i);
+
+ tx.rollback();
+ }
+
+ assertEquals(size, cache.size());
+ }
+ }
+
+ // Check rollback update.
+ for (int i = 0; i < KEYS; i++) {
+ final Integer key = i;
+
+ try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.put(key, -1);
+
+ tx.rollback();
+ }
+
+ assertEquals(size, cache.size());
+ }
+
+ // Check rollback remove.
+ for (int i = 0; i < KEYS; i++) {
+ final Integer key = i;
+
+ try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
+ cache.remove(key);
+
+ tx.rollback();
+ }
+
+ assertEquals(size, cache.size());
+ }
+
+ // Restore original state.
for (int i = 0; i < KEYS; i++) {
if (i % 2 == 0) {
final Integer key = i;
@@ -3222,6 +3084,10 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
assertEquals(size, cache.size());
}
}
+
+ // Check state.
+ for (int i = 0; i < KEYS; i++)
+ assertEquals(i, cache.get(i));
}
/**
@@ -3467,7 +3333,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @param readModes Read modes to check.
* @return Value.
*/
- private Object checkAndGet(boolean inTx, IgniteCache cache, Object key, ReadMode ... readModes) {
+ private Object checkAndGet(boolean inTx, IgniteCache cache, Object key, ReadMode... readModes) {
assert readModes != null && readModes.length > 0;
if (inTx)
@@ -3531,13 +3397,12 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
}
}
-
/**
* Checks values obtained with different read modes.
* And returns value in case of it's equality for all read modes.
* Do not use in tests with writers contention.
*
- * // TODO remove inTx flag in IGNITE-7764
+ * // TODO remove inTx flag in IGNITE-6938
* @param inTx Flag whether current read is inside transaction.
* This is because reads can't see writes made in current transaction.
* @param cache Cache.
@@ -3545,7 +3410,7 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
* @param readModes Read modes to check.
* @return Value.
*/
- private Map checkAndGetAll(boolean inTx, IgniteCache cache, Set keys, ReadMode ... readModes) {
+ private Map checkAndGetAll(boolean inTx, IgniteCache cache, Set keys, ReadMode... readModes) {
assert readModes != null && readModes.length > 0;
if (inTx)
@@ -3571,11 +3436,10 @@ public class CacheMvccTransactionsTest extends CacheMvccAbstractTest {
return prevVal;
}
-
/**
* Reads value from cache for the given key using given read mode.
*
- * // TODO IGNITE-7764 remove inTx flag
+ * // TODO IGNITE-6938 remove inTx flag
* // TODO IGNITE-6739 add SQL-get support "select _key, _val from cache where _key in ... keySet"
* @param inTx Flag whether current read is inside transaction.
* This is because reads can't see writes made in current transaction.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
index d7948bd..381d9a9 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/datastreamer/DataStreamProcessorMvccSelfTest.java
@@ -76,6 +76,11 @@ public class DataStreamProcessorMvccSelfTest extends DataStreamProcessorSelfTest
}
/** {@inheritDoc} */
+ @Override public void testFlushTimeout() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9321");
+ }
+
+ /** {@inheritDoc} */
@Override public void testLocal() throws Exception {
// Do not check local caches with MVCC enabled.
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/testframework/configvariations/ConfigVariations.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/configvariations/ConfigVariations.java b/modules/core/src/test/java/org/apache/ignite/testframework/configvariations/ConfigVariations.java
index ca22b56..1b85d33 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/configvariations/ConfigVariations.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/configvariations/ConfigVariations.java
@@ -42,7 +42,6 @@ import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.configuration.TopologyValidator;
import org.apache.ignite.internal.binary.BinaryMarshaller;
import org.apache.ignite.internal.processors.cache.MapCacheStoreStrategy;
-import org.apache.ignite.internal.marshaller.optimized.OptimizedMarshaller;
import static org.apache.ignite.internal.util.lang.GridFunc.asArray;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DhtResultSetEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DhtResultSetEnlistFuture.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DhtResultSetEnlistFuture.java
index 1d382f7..7f22107 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DhtResultSetEnlistFuture.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/DhtResultSetEnlistFuture.java
@@ -20,8 +20,8 @@ package org.apache.ignite.internal.processors.query.h2;
import java.sql.ResultSet;
import java.util.UUID;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
-import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxAbstractEnlistFuture;
import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxLocalAdapter;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxQueryAbstractEnlistFuture;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.processors.query.UpdateSourceIterator;
@@ -31,7 +31,7 @@ import org.jetbrains.annotations.Nullable;
/**
*
*/
-public class DhtResultSetEnlistFuture extends GridDhtTxAbstractEnlistFuture implements ResultSetEnlistFuture {
+public class DhtResultSetEnlistFuture extends GridDhtTxQueryAbstractEnlistFuture implements ResultSetEnlistFuture {
/** */
private ResultSet rs;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/NearResultSetEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/NearResultSetEnlistFuture.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/NearResultSetEnlistFuture.java
index 1856430..968a856 100644
--- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/NearResultSetEnlistFuture.java
+++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/NearResultSetEnlistFuture.java
@@ -32,9 +32,6 @@ import org.jetbrains.annotations.Nullable;
*
*/
public class NearResultSetEnlistFuture extends GridNearTxQueryResultsEnlistFuture implements ResultSetEnlistFuture {
- /** */
- private static final long serialVersionUID = 877907044489718378L;
-
/**
* @param nearNodeId Near node ID.
* @param nearLockVer Near lock version.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
index 8c6f407..76f8013 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/index/SqlTransactionsCommandsWithMvccEnabledSelfTest.java
@@ -132,6 +132,8 @@ public class SqlTransactionsCommandsWithMvccEnabledSelfTest extends AbstractSche
* Test that attempting to perform various SQL operations within non SQL transaction yields an exception.
*/
public void testSqlOperationsWithinNonSqlTransaction() {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
+
assertSqlOperationWithinNonSqlTransactionThrows("COMMIT");
assertSqlOperationWithinNonSqlTransactionThrows("ROLLBACK");
@@ -230,8 +232,8 @@ public class SqlTransactionsCommandsWithMvccEnabledSelfTest extends AbstractSche
return null;
}
- }, IgniteCheckedException.class,
- "SQL queries and cache operations may not be used in the same transaction.");
+ }, UnsupportedOperationException.class,
+ "operations are not supported on transactional caches when MVCC is enabled.");
}
finally {
try {
@@ -266,78 +268,6 @@ public class SqlTransactionsCommandsWithMvccEnabledSelfTest extends AbstractSche
*/
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
public void testCacheOperationsFromSqlTransaction() {
- checkCacheOperationThrows("get", 1);
-
- checkCacheOperationThrows("getAsync", 1);
-
- checkCacheOperationThrows("getEntry", 1);
-
- checkCacheOperationThrows("getEntryAsync", 1);
-
- checkCacheOperationThrows("getAndPut", 1, 1);
-
- checkCacheOperationThrows("getAndPutAsync", 1, 1);
-
- checkCacheOperationThrows("getAndPutIfAbsent", 1, 1);
-
- checkCacheOperationThrows("getAndPutIfAbsentAsync", 1, 1);
-
- checkCacheOperationThrows("getAndReplace", 1, 1);
-
- checkCacheOperationThrows("getAndReplaceAsync", 1, 1);
-
- checkCacheOperationThrows("getAndRemove", 1);
-
- checkCacheOperationThrows("getAndRemoveAsync", 1);
-
- checkCacheOperationThrows("containsKey", 1);
-
- checkCacheOperationThrows("containsKeyAsync", 1);
-
- checkCacheOperationThrows("put", 1, 1);
-
- checkCacheOperationThrows("putAsync", 1, 1);
-
- checkCacheOperationThrows("putIfAbsent", 1, 1);
-
- checkCacheOperationThrows("putIfAbsentAsync", 1, 1);
-
- checkCacheOperationThrows("remove", 1);
-
- checkCacheOperationThrows("removeAsync", 1);
-
- checkCacheOperationThrows("remove", 1, 1);
-
- checkCacheOperationThrows("removeAsync", 1, 1);
-
- checkCacheOperationThrows("replace", 1, 1);
-
- checkCacheOperationThrows("replaceAsync", 1, 1);
-
- checkCacheOperationThrows("replace", 1, 1, 1);
-
- checkCacheOperationThrows("replaceAsync", 1, 1, 1);
-
- checkCacheOperationThrows("getAll", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("containsKeys", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("getEntries", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("putAll", Collections.singletonMap(1, 1));
-
- checkCacheOperationThrows("removeAll", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("getAllAsync", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("containsKeysAsync", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("getEntriesAsync", new HashSet<>(Arrays.asList(1, 2)));
-
- checkCacheOperationThrows("putAllAsync", Collections.singletonMap(1, 1));
-
- checkCacheOperationThrows("removeAllAsync", new HashSet<>(Arrays.asList(1, 2)));
-
checkCacheOperationThrows("invoke", 1, ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
checkCacheOperationThrows("invoke", 1, CACHE_ENTRY_PROC, X.EMPTY_OBJECT_ARRAY);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryAbstractTest.java
index 5c81974..00c748e 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSelectForUpdateQueryAbstractTest.java
@@ -96,6 +96,8 @@ public abstract class CacheMvccSelectForUpdateQueryAbstractTest extends CacheMvc
*
*/
public void testSelectForUpdateDistributed() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9724");
+
doTestSelectForUpdateDistributed("Person", false);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
index 796c0bb..4ea53e0 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlQueriesAbstractTest.java
@@ -151,6 +151,8 @@ public abstract class CacheMvccSqlQueriesAbstractTest extends CacheMvccAbstractT
* @throws Exception If failed.
*/
private void updateSingleValue(boolean singleNode, final boolean locQry) throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9540");
+
final int VALS = 100;
final int writers = 4;
@@ -377,6 +379,8 @@ public abstract class CacheMvccSqlQueriesAbstractTest extends CacheMvccAbstractT
* @throws Exception If failed.
*/
private void joinTransactional(boolean singleNode, final boolean distributedJoin) throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
+
final int KEYS = 100;
final int writers = 4;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java
index b881f02..7076362 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccSqlTxQueriesAbstractTest.java
@@ -1120,6 +1120,8 @@ public abstract class CacheMvccSqlTxQueriesAbstractTest extends CacheMvccAbstrac
* @throws Exception If failed.
*/
public void testQueryInsertUpdateMultithread() throws Exception {
+ fail("https://issues.apache.org/jira/browse/IGNITE-9470");
+
ccfg = cacheConfiguration(cacheMode(), FULL_SYNC, 2, DFLT_PARTITION_COUNT)
.setIndexedTypes(Integer.class, Integer.class);
@@ -1196,14 +1198,17 @@ public abstract class CacheMvccSqlTxQueriesAbstractTest extends CacheMvccAbstrac
}
}, 1));
- fut.markInitialized();
-
try {
+ fut.markInitialized();
+
fut.get(TX_TIMEOUT);
}
catch (IgniteCheckedException e) {
onException(ex, e);
}
+ finally {
+ phaser.forceTermination();
+ }
Exception ex0 = ex.get();
@@ -1248,7 +1253,7 @@ public abstract class CacheMvccSqlTxQueriesAbstractTest extends CacheMvccAbstrac
try (Transaction tx = node.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
tx.timeout(TX_TIMEOUT);
- barrier.await();
+ barrier.await(TX_TIMEOUT, TimeUnit.MILLISECONDS);
IgniteCache<Object, Object> cache0 = node.cache(DEFAULT_CACHE_NAME);
@@ -1262,7 +1267,7 @@ public abstract class CacheMvccSqlTxQueriesAbstractTest extends CacheMvccAbstrac
}
}
- barrier.await();
+ barrier.await(TX_TIMEOUT, TimeUnit.MILLISECONDS);
qry = new SqlFieldsQuery("UPDATE Integer SET _val = (_key * 10)");
@@ -1820,7 +1825,7 @@ public abstract class CacheMvccSqlTxQueriesAbstractTest extends CacheMvccAbstrac
do {
p = phaser.arriveAndAwaitAdvance();
}
- while (p < phase);
+ while (p < phase && p >= 0 /* check termination */ );
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadBulkOpsTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadBulkOpsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadBulkOpsTest.java
new file mode 100644
index 0000000..46aeaa1
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadBulkOpsTest.java
@@ -0,0 +1,441 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.mvcc;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.cache.CacheMode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.GET;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.DML;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.PUT;
+
+/**
+ * Test basic mvcc bulk cache operations.
+ */
+public class MvccRepeatableReadBulkOpsTest extends CacheMvccAbstractTest {
+ /** {@inheritDoc} */
+ @Override protected CacheMode cacheMode() {
+ return CacheMode.PARTITIONED;
+ }
+
+ /** */
+ private int nodesCount() {
+ return 4;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTestsStarted() throws Exception {
+ super.beforeTestsStarted();
+
+ cleanPersistenceDir();
+
+ startGridsMultiThreaded(nodesCount() - 1);
+
+ client = true;
+
+ startGrid(nodesCount() - 1);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTestsStopped() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+
+ super.afterTestsStopped();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ grid(0).createCache(cacheConfiguration(cacheMode(), FULL_SYNC, 1, 32).
+ setIndexedTypes(Integer.class, MvccTestAccount.class));
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ grid(0).destroyCache(DEFAULT_CACHE_NAME);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationGetPut() throws Exception {
+ checkOperations(GET, GET, PUT, true);
+ checkOperations(GET, GET, PUT, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationSqlPut() throws Exception {
+ checkOperations(SQL, SQL, PUT, true);
+ checkOperations(SQL, SQL, PUT, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationSqlDml() throws Exception {
+ checkOperations(SQL, SQL, DML, true);
+ checkOperations(SQL, SQL, DML, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationGetDml() throws Exception {
+ checkOperations(GET, GET, DML, true);
+ checkOperations(GET, GET, DML, false);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationMixedPut() throws Exception {
+ checkOperations(SQL, GET, PUT, false);
+ checkOperations(SQL, GET, PUT, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationMixedPut2() throws Exception {
+ checkOperations(GET, SQL, PUT, false);
+ checkOperations(GET, SQL, PUT, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationMixedDml() throws Exception {
+ checkOperations(SQL, GET, DML, false);
+ checkOperations(SQL, GET, DML, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testRepeatableReadIsolationMixedDml2() throws Exception {
+ checkOperations(GET, SQL, DML, false);
+ checkOperations(GET, SQL, DML, true);
+ }
+
+ /**
+ * @throws Exception If failed.
+ */
+ public void testOperationConsistency() throws Exception {
+ checkOperationsConsistency(PUT, false);
+ checkOperationsConsistency(DML, false);
+ checkOperationsConsistency(PUT, true);
+ checkOperationsConsistency(DML, true);
+ }
+
+ /**
+ * Checks SQL and CacheAPI operation isolation consistency.
+ *
+ * @param readModeBefore read mode used before value updated.
+ * @param readModeBefore read mode used after value updated.
+ * @param writeMode write mode used for update.
+ * @throws Exception If failed.
+ */
+ private void checkOperations(ReadMode readModeBefore, ReadMode readModeAfter,
+ WriteMode writeMode, boolean readFromClient) throws Exception {
+ Ignite node1 = grid(readFromClient ? nodesCount() - 1 : 0);
+ Ignite node2 = grid(readFromClient ? 0 : nodesCount() - 1);
+
+ TestCache<Integer, MvccTestAccount> cache1 = new TestCache<>(node1.cache(DEFAULT_CACHE_NAME));
+ TestCache<Integer, MvccTestAccount> cache2 = new TestCache<>(node2.cache(DEFAULT_CACHE_NAME));
+
+ final Set<Integer> keysForUpdate = new HashSet<>(3);
+ final Set<Integer> keysForRemove = new HashSet<>(3);
+
+ final Set<Integer> allKeys = generateKeySet(grid(0).cache(DEFAULT_CACHE_NAME), keysForUpdate, keysForRemove);
+
+ final Map<Integer, MvccTestAccount> initialMap = allKeys.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 1)));
+
+ final Map<Integer, MvccTestAccount> updateMap = keysForUpdate.stream().collect(Collectors.toMap(Function.identity(),
+ k -> new MvccTestAccount(k, 2))); /* Removed keys are excluded. */
+
+ cache1.cache.putAll(initialMap);
+
+ IgniteTransactions txs1 = node1.transactions();
+ IgniteTransactions txs2 = node2.transactions();
+
+ CountDownLatch updateStart = new CountDownLatch(1);
+ CountDownLatch updateFinish = new CountDownLatch(1);
+
+ // Start concurrent transactions and check isolation.
+ IgniteInternalFuture<Void> updater = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ updateStart.await();
+
+ try (Transaction tx = txs2.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+
+ updateEntries(cache2, updateMap, writeMode);
+ removeEntries(cache2, keysForRemove, writeMode);
+
+ checkContains(cache2, true, updateMap.keySet());
+ checkContains(cache2, false, keysForRemove);
+
+ assertEquals(updateMap, cache2.cache.getAll(allKeys));
+
+ tx.commit();
+ }
+
+ updateFinish.countDown();
+
+ return null;
+ }
+ });
+
+ IgniteInternalFuture<Void> reader = GridTestUtils.runAsync(new Callable<Void>() {
+ @Override public Void call() throws Exception {
+ try (Transaction tx = txs1.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ assertEquals(initialMap, getEntries(cache1, allKeys, readModeBefore));
+
+ checkContains(cache1, true, allKeys);
+
+ updateStart.countDown();
+ updateFinish.await();
+
+ assertEquals(initialMap, getEntries(cache1, allKeys, readModeAfter));
+
+ checkContains(cache1, true,allKeys);
+
+ tx.commit();
+ }
+
+ return null;
+ }
+ });
+
+ try {
+ updater.get(3_000, TimeUnit.MILLISECONDS);
+ reader.get(3_000, TimeUnit.MILLISECONDS);
+ }
+ catch (Throwable e) {
+ throw new AssertionError(e);
+ }
+ finally {
+ updateStart.countDown();
+ updateFinish.countDown();
+ }
+
+ assertEquals(updateMap, cache1.cache.getAll(allKeys));
+ }
+
+ /**
+ * Generate 2 sets of keys. Each set contains primary, backup and non-affinity key for given node cache.
+ *
+ * @param cache Cache.
+ * @param keySet1 Key set.
+ * @param keySet2 Key set.
+ * @return All keys.
+ * @throws IgniteCheckedException If failed.
+ */
+ protected Set<Integer> generateKeySet(IgniteCache<Object, Object> cache, Set<Integer> keySet1,
+ Set<Integer> keySet2) throws IgniteCheckedException {
+ LinkedHashSet<Integer> allKeys = new LinkedHashSet<>();
+
+ allKeys.addAll(primaryKeys(cache, 2));
+ allKeys.addAll(backupKeys(cache, 2, 1));
+ allKeys.addAll(nearKeys(cache, 2, 1));
+
+ List<Integer> keys0 = new ArrayList<>(allKeys);
+
+ for (int i = 0; i < 6; i++) {
+ if (i % 2 == 0)
+ keySet1.add(keys0.get(i));
+ else
+ keySet2.add(keys0.get(i));
+ }
+
+ assert allKeys.size() == 6; // Expects no duplicates.
+
+ return allKeys;
+ }
+
+ /**
+ * Checks SQL and CacheAPI operation see consistent results before and after update.
+ *
+ * @throws Exception If failed.
+ */
+ private void checkOperationsConsistency(WriteMode writeMode, boolean requestFromClient) throws Exception {
+ Ignite node = grid(requestFromClient ? nodesCount() - 1 : 0);
+
+ TestCache<Integer, MvccTestAccount> cache = new TestCache<>(node.cache(DEFAULT_CACHE_NAME));
+
+ final Set<Integer> keysForUpdate = new HashSet<>(3);
+ final Set<Integer> keysForRemove = new HashSet<>(3);
+
+ final Set<Integer> allKeys = generateKeySet(grid(0).cache(DEFAULT_CACHE_NAME), keysForUpdate, keysForRemove);
+
+ int updCnt = 1;
+
+ final Map<Integer, MvccTestAccount> initialVals = allKeys.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 1)));
+
+ cache.cache.putAll(initialVals);
+
+ IgniteTransactions txs = node.transactions();
+
+ Map<Integer, MvccTestAccount> updatedVals = null;
+
+ try (Transaction tx = txs.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ Map<Integer, MvccTestAccount> vals1 = getEntries(cache, allKeys, GET);
+ Map<Integer, MvccTestAccount> vals2 = getEntries(cache, allKeys, SQL);
+
+ assertEquals(initialVals, vals1);
+ assertEquals(initialVals, vals2);
+
+ for (ReadMode readMode : new ReadMode[] {GET, SQL}) {
+ int updCnt0 = ++updCnt;
+
+ updatedVals = keysForUpdate.stream().collect(Collectors.toMap(Function.identity(),
+ k -> new MvccTestAccount(k, updCnt0)));
+
+ updateEntries(cache, updatedVals, writeMode);
+ removeEntries(cache, keysForRemove, writeMode);
+
+ assertEquals(String.valueOf(readMode), updatedVals, getEntries(cache, allKeys, readMode));
+ }
+
+ tx.commit();
+ }
+
+ try (Transaction tx = txs.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ assertEquals(updatedVals, getEntries(cache, allKeys, GET));
+ assertEquals(updatedVals, getEntries(cache, allKeys, SQL));
+
+ tx.commit();
+ }
+ }
+
+ /**
+ * Gets values with given read mode.
+ *
+ * @param cache Cache.
+ * @param keys Key to be read.
+ * @param readMode Read mode.
+ * @return Key-value result map.
+ */
+ protected Map<Integer, MvccTestAccount> getEntries(
+ TestCache<Integer, MvccTestAccount> cache,
+ Set<Integer> keys,
+ ReadMode readMode) {
+ switch (readMode) {
+ case GET:
+ return cache.cache.getAll(keys);
+ case SQL:
+ return getAllSql(cache);
+ default:
+ fail();
+ }
+
+ return null;
+ }
+
+ /**
+ * Updates entries with given write mode.
+ *
+ * @param cache Cache.
+ * @param entries Entries to be updated.
+ * @param writeMode Write mode.
+ */
+ protected void updateEntries(
+ TestCache<Integer, MvccTestAccount> cache,
+ Map<Integer, MvccTestAccount> entries,
+ WriteMode writeMode) {
+ switch (writeMode) {
+ case PUT: {
+ cache.cache.putAll(entries);
+
+ break;
+ }
+ case DML: {
+ for (Map.Entry<Integer, MvccTestAccount> e : entries.entrySet())
+ mergeSql(cache, e.getKey(), e.getValue().val, e.getValue().updateCnt);
+
+ break;
+ }
+ default:
+ fail();
+ }
+ }
+
+ /**
+ * Updates entries with given write mode.
+ *
+ * @param cache Cache.
+ * @param keys Key to be deleted.
+ * @param writeMode Write mode.
+ */
+ protected void removeEntries(
+ TestCache<Integer, MvccTestAccount> cache,
+ Set<Integer> keys,
+ WriteMode writeMode) {
+ switch (writeMode) {
+ case PUT: {
+ cache.cache.removeAll(keys);
+
+ break;
+ }
+ case DML: {
+ for (Integer key : keys)
+ removeSql(cache, key);
+
+ break;
+ }
+ default:
+ fail();
+ }
+ }
+
+ /**
+ * Check cache contains entries.
+ *
+ * @param cache Cache.
+ * @param expected Expected result.
+ * @param keys Keys to check.
+ */
+ protected void checkContains(TestCache<Integer, MvccTestAccount> cache, boolean expected, Set<Integer> keys) {
+ assertEquals(expected, cache.cache.containsKeys(keys));
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadOperationsTest.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadOperationsTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadOperationsTest.java
new file mode 100644
index 0000000..c782f98
--- /dev/null
+++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/MvccRepeatableReadOperationsTest.java
@@ -0,0 +1,276 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.mvcc;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import org.apache.ignite.Ignite;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteTransactions;
+import org.apache.ignite.transactions.Transaction;
+import org.apache.ignite.transactions.TransactionConcurrency;
+import org.apache.ignite.transactions.TransactionIsolation;
+
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.GET;
+import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL;
+
+/**
+ * Test basic mvcc cache operation operations.
+ */
+public class MvccRepeatableReadOperationsTest extends MvccRepeatableReadBulkOpsTest {
+ /** {@inheritDoc} */
+ @Override protected Map<Integer, MvccTestAccount> getEntries(
+ TestCache<Integer, MvccTestAccount> cache,
+ Set<Integer> keys,
+ ReadMode readMode) {
+
+ switch (readMode) {
+ case GET: {
+ Map<Integer, MvccTestAccount> res = new HashMap<>();
+
+ for (Integer key : keys) {
+ MvccTestAccount val = cache.cache.get(key);
+
+ if(val != null)
+ res.put(key, val);
+ }
+
+ return res;
+ }
+ case SQL:
+ return getAllSql(cache);
+ default:
+ fail();
+ }
+
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ protected void updateEntries(
+ TestCache<Integer, MvccTestAccount> cache,
+ Map<Integer, MvccTestAccount> entries,
+ WriteMode writeMode) {
+ switch (writeMode) {
+ case PUT: {
+ for (Map.Entry<Integer, MvccTestAccount> e : entries.entrySet())
+ if (e.getValue() == null)
+ cache.cache.remove(e.getKey());
+ else
+ cache.cache.put(e.getKey(), e.getValue());
+
+ break;
+ }
+ case DML: {
+ for (Map.Entry<Integer, MvccTestAccount> e : entries.entrySet()) {
+ if (e.getValue() == null)
+ removeSql(cache, e.getKey());
+ else
+ mergeSql(cache, e.getKey(), e.getValue().val, e.getValue().updateCnt);
+ }
+ break;
+ }
+ default:
+ fail();
+ }
+ }
+
+ /** {@inheritDoc} */
+ protected void removeEntries(
+ TestCache<Integer, MvccTestAccount> cache,
+ Set<Integer> keys,
+ WriteMode writeMode) {
+ switch (writeMode) {
+ case PUT: {
+ for (Integer key : keys)
+ cache.cache.remove(key);
+
+ break;
+ }
+ case DML: {
+ for (Integer key : keys)
+ removeSql(cache, key);
+
+ break;
+ }
+ default:
+ fail();
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void checkContains(TestCache<Integer, MvccTestAccount> cache, boolean expected,
+ Set<Integer> keys) {
+ for (Integer key : keys)
+ assertEquals(expected, cache.cache.containsKey(key));
+ }
+
+ /**
+ * Check getAndPut/getAndRemove operations consistency.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testGetAndUpdateOperations() throws IgniteCheckedException {
+ Ignite node1 = grid(0);
+
+ TestCache<Integer, MvccTestAccount> cache1 = new TestCache<>(node1.cache(DEFAULT_CACHE_NAME));
+
+ final Set<Integer> keysForUpdate = new HashSet<>(3);
+ final Set<Integer> keysForRemove = new HashSet<>(3);
+
+ final Set<Integer> allKeys = generateKeySet(grid(0).cache(DEFAULT_CACHE_NAME), keysForUpdate, keysForRemove);
+
+ final Map<Integer, MvccTestAccount> initialMap = keysForRemove.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 1)));
+
+ Map<Integer, MvccTestAccount> updateMap = keysForUpdate.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 3)));
+
+ cache1.cache.putAll(initialMap);
+
+ IgniteTransactions txs = node1.transactions();
+ try (Transaction tx = txs.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ for (Integer key : keysForUpdate) {
+ MvccTestAccount newVal1 = new MvccTestAccount(key, 1);
+
+ assertNull(cache1.cache.getAndPut(key, newVal1)); // Check create.
+
+ MvccTestAccount newVal2 = new MvccTestAccount(key, 2);
+
+ assertEquals(newVal1, cache1.cache.getAndPut(key, newVal2)); // Check update.
+ }
+
+ for (Integer key : keysForRemove) {
+ assertEquals(initialMap.get(key), cache1.cache.getAndRemove(key)); // Check remove existed.
+
+ assertNull(cache1.cache.getAndRemove(key)); // Check remove non-existed.
+ }
+
+ for (Integer key : allKeys) {
+ MvccTestAccount oldVal = new MvccTestAccount(key, 2);
+ MvccTestAccount newVal = new MvccTestAccount(key, 3);
+
+ if (keysForRemove.contains(key))
+ assertNull(cache1.cache.getAndReplace(key, newVal)); // Omit update 'null'.
+ else
+ assertEquals(oldVal, cache1.cache.getAndReplace(key, newVal)); // Check updated.
+ }
+
+ assertEquals(updateMap, getEntries(cache1, allKeys, SQL));
+ assertEquals(updateMap, getEntries(cache1, allKeys, GET));
+
+ tx.commit();
+ }
+
+ assertEquals(updateMap, getEntries(cache1, allKeys, SQL));
+ assertEquals(updateMap, getEntries(cache1, allKeys, GET));
+ }
+
+ /**
+ * Check getAndPut/getAndRemove operations consistency.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testPutIfAbsentConsistency() throws IgniteCheckedException {
+ Ignite node1 = grid(0);
+
+ TestCache<Integer, MvccTestAccount> cache1 = new TestCache<>(node1.cache(DEFAULT_CACHE_NAME));
+
+ final Set<Integer> keysForCreate = new HashSet<>(3);
+ final Set<Integer> keysForUpdate = new HashSet<>(3);
+
+ final Set<Integer> allKeys = generateKeySet(grid(0).cache(DEFAULT_CACHE_NAME), keysForCreate, keysForUpdate);
+
+ final Map<Integer, MvccTestAccount> initialMap = keysForUpdate.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 1)));
+
+ Map<Integer, MvccTestAccount> updatedMap = allKeys.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 1)));
+
+ cache1.cache.putAll(initialMap);
+
+ IgniteTransactions txs = node1.transactions();
+ try (Transaction tx = txs.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ for (Integer key : keysForUpdate)
+ assertFalse(cache1.cache.putIfAbsent(key, new MvccTestAccount(key, 2))); // Check update.
+
+ for (Integer key : keysForCreate)
+ assertTrue(cache1.cache.putIfAbsent(key, new MvccTestAccount(key, 1))); // Check create.
+
+ assertEquals(updatedMap, getEntries(cache1, allKeys, SQL));
+
+ tx.commit();
+ }
+
+ assertEquals(updatedMap, getEntries(cache1, allKeys, SQL));
+ assertEquals(updatedMap, getEntries(cache1, allKeys, GET));
+ }
+
+ /**
+ * Check getAndPut/getAndRemove operations consistency.
+ *
+ * @throws IgniteCheckedException If failed.
+ */
+ public void testReplaceConsistency() throws IgniteCheckedException {
+ Ignite node1 = grid(0);
+
+ TestCache<Integer, MvccTestAccount> cache1 = new TestCache<>(node1.cache(DEFAULT_CACHE_NAME));
+
+ final Set<Integer> existedKeys = new HashSet<>(3);
+ final Set<Integer> nonExistedKeys = new HashSet<>(3);
+
+ final Set<Integer> allKeys = generateKeySet(grid(0).cache(DEFAULT_CACHE_NAME), existedKeys, nonExistedKeys);
+
+ final Map<Integer, MvccTestAccount> initialMap = existedKeys.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 1)));
+
+ Map<Integer, MvccTestAccount> updateMap = existedKeys.stream().collect(
+ Collectors.toMap(k -> k, k -> new MvccTestAccount(k, 3)));
+
+ cache1.cache.putAll(initialMap);
+
+ IgniteTransactions txs = node1.transactions();
+ try (Transaction tx = txs.txStart(TransactionConcurrency.PESSIMISTIC, TransactionIsolation.REPEATABLE_READ)) {
+ for (Integer key : allKeys) {
+ MvccTestAccount newVal = new MvccTestAccount(key, 2);
+
+ if(existedKeys.contains(key)) {
+ assertTrue(cache1.cache.replace(key, new MvccTestAccount(key, 1), newVal));
+
+ assertEquals(newVal, cache1.cache.getAndReplace(key, new MvccTestAccount(key, 3)));
+ }
+ else {
+ assertFalse(cache1.cache.replace(key, new MvccTestAccount(key, 1), newVal));
+
+ assertNull(cache1.cache.getAndReplace(key, new MvccTestAccount(key, 3)));
+ }
+ }
+
+ assertEquals(updateMap, getEntries(cache1, allKeys, SQL));
+ assertEquals(updateMap, getEntries(cache1, allKeys, GET));
+
+ tx.commit();
+ }
+
+ assertEquals(updateMap, getEntries(cache1, allKeys, SQL));
+ assertEquals(updateMap, getEntries(cache1, allKeys, GET));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
index 726c4e9..21ab2e6 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgniteCacheMvccSqlTestSuite.java
@@ -42,6 +42,8 @@ import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSqlConfiguratio
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSqlUpdateCountersTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccSqlLockTimeoutTest;
import org.apache.ignite.internal.processors.cache.mvcc.CacheMvccStreamingInsertTest;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccRepeatableReadBulkOpsTest;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccRepeatableReadOperationsTest;
import org.apache.ignite.internal.processors.query.h2.GridIndexRebuildWithMvccEnabledSelfTest;
/**
@@ -64,6 +66,10 @@ public class IgniteCacheMvccSqlTestSuite extends TestSuite {
suite.addTestSuite(GridIndexRebuildWithMvccEnabledSelfTest.class);
+ // SQL vs CacheAPI consistency.
+ suite.addTestSuite(MvccRepeatableReadOperationsTest.class);
+ suite.addTestSuite(MvccRepeatableReadBulkOpsTest.class);
+
// JDBC tests.
suite.addTestSuite(CacheMvccSizeWithConcurrentJdbcTransactionTest.class);
suite.addTestSuite(CacheMvccScanQueryWithConcurrentJdbcTransactionTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
index 1f600dd..5b4106a 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/ApiParity/IgniteConfigurationParityTest.cs
@@ -81,9 +81,8 @@ namespace Apache.Ignite.Core.Tests.ApiParity
"TimeServerPortRange",
"IncludeProperties",
"isAutoActivationEnabled", // IGNITE-7301
- "isMvccEnabled", //TODO: IGNITE-9390: Remove when Mvcc support will be added.
- "MvccVacuumTimeInterval", //TODO: IGNITE-9390: Remove when Mvcc support will be added.
- "MvccVacuumThreadCnt" //TODO: IGNITE-9390: Remove when Mvcc support will be added.
+ "MvccVacuumFrequency", //TODO: IGNITE-9390: Remove when Mvcc support will be added.
+ "MvccVacuumThreadCount" //TODO: IGNITE-9390: Remove when Mvcc support will be added.
};
/// <summary>
[14/21] ignite git commit: IGNITE-9713: [ML] Fix JavaDocs in ML
Prerpocessing package
Posted by sb...@apache.org.
IGNITE-9713: [ML] Fix JavaDocs in ML Prerpocessing package
this closes #4867
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a373486f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a373486f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a373486f
Branch: refs/heads/ignite-gg-14206
Commit: a373486f817e9c40e2eb0748b28c1d7df14e4203
Parents: f7f834b
Author: zaleslaw <za...@gmail.com>
Authored: Fri Sep 28 16:01:35 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Sep 28 16:01:35 2018 +0300
----------------------------------------------------------------------
.../ignite/ml/knn/ann/ANNClassificationModel.java | 2 +-
.../org/apache/ignite/ml/knn/ann/ANNModelFormat.java | 2 +-
.../preprocessing/encoding/EncoderPreprocessor.java | 2 ++
.../ml/preprocessing/encoding/EncoderTrainer.java | 4 +---
.../stringencoder/StringEncoderPreprocessor.java | 14 ++++++++++++++
5 files changed, 19 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a373486f/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
index bec82a9..6ef3990 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNClassificationModel.java
@@ -50,7 +50,7 @@ public class ANNClassificationModel extends NNClassificationModel {
/**
* Build the model based on a candidates set.
* @param centers The candidates set.
- * @param centroindsStat
+ * @param centroindsStat The stat about centroids.
*/
public ANNClassificationModel(LabeledVectorSet<ProbableLabel, LabeledVector> centers,
ANNClassificationTrainer.CentroidStat centroindsStat) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/a373486f/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
index be09828..e0c2c3a 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/knn/ann/ANNModelFormat.java
@@ -41,7 +41,7 @@ public class ANNModelFormat extends KNNModelFormat implements Serializable {
* @param k Amount of nearest neighbors.
* @param measure Distance measure.
* @param stgy kNN strategy.
- * @param candidatesStat
+ * @param candidatesStat The stat about candidates.
*/
public ANNModelFormat(int k,
DistanceMeasure measure,
http://git-wip-us.apache.org/repos/asf/ignite/blob/a373486f/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
index 7df44f3..5c60da9 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderPreprocessor.java
@@ -25,6 +25,8 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
/**
* Preprocessing function that makes encoding.
*
+ * This a base abstract class that keeps the common fields for all child encoding preprocessors.
+ *
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/a373486f/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
index 8b2d9b7..9a97a6d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/EncoderTrainer.java
@@ -35,9 +35,7 @@ import org.apache.ignite.ml.preprocessing.encoding.stringencoder.StringEncoderPr
import org.jetbrains.annotations.NotNull;
/**
- * Trainer of the String Encoder preprocessor.
- * The String Encoder encodes string values (categories) to double values in range [0.0, amountOfCategories)
- * where the most popular value will be presented as 0.0 and the least popular value presented with amountOfCategories-1 value.
+ * Trainer of the String Encoder and One-Hot Encoder preprocessors.
*
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
http://git-wip-us.apache.org/repos/asf/ignite/blob/a373486f/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
index 12f98f6..c2474ef 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/preprocessing/encoding/stringencoder/StringEncoderPreprocessor.java
@@ -28,6 +28,20 @@ import org.apache.ignite.ml.preprocessing.encoding.EncoderPreprocessor;
/**
* Preprocessing function that makes String encoding.
*
+ * The String Encoder Preprocessor encodes string values (categories) to double values
+ * in range [0.0, amountOfCategories), where the most popular value will be presented as 0.0 and
+ * the least popular value presented with amountOfCategories-1 value.
+ * <p>
+ * This preprocessor can transform multiple columns which indices are handled during training process. These indexes could be defined via .withEncodedFeature(featureIndex) call.
+ * </p>
+ * <p>
+ * NOTE: it doesn’t add new column but change data in-place.
+ *</p>
+ * <p>
+ * There is only a one strategy regarding how StringEncoder will handle unseen labels
+ * when you have fit a StringEncoder on one dataset and then use it to transform another:
+ * put unseen labels in a special additional bucket, at index is equal amountOfCategories.
+ * </p>
* @param <K> Type of a key in {@code upstream} data.
* @param <V> Type of a value in {@code upstream} data.
*/
[11/21] ignite git commit: IGNITE-7764: MVCC: cache API support. This
closes #4725.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
index b8c78bd..9493510 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxLocal.java
@@ -62,7 +62,6 @@ import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxPrep
import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtDetachedCacheEntry;
import org.apache.ignite.internal.processors.cache.dr.GridCacheDrInfo;
import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker;
-import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTrackerImpl;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils;
import org.apache.ignite.internal.processors.cache.transactions.IgniteInternalTx;
@@ -72,6 +71,7 @@ import org.apache.ignite.internal.processors.cache.transactions.TransactionProxy
import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyImpl;
import org.apache.ignite.internal.processors.cache.transactions.TransactionProxyRollbackOnlyImpl;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.query.EnlistOperation;
import org.apache.ignite.internal.processors.query.UpdateSourceIterator;
import org.apache.ignite.internal.processors.timeout.GridTimeoutObject;
import org.apache.ignite.internal.transactions.IgniteTxOptimisticCheckedException;
@@ -95,6 +95,7 @@ import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiClosure;
+import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteInClosure;
import org.apache.ignite.lang.IgniteUuid;
@@ -191,11 +192,11 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
/** */
private MvccQueryTracker mvccTracker;
- /** Whether this transaction is for SQL operations or not.<p>
+ /** Whether this is Mvcc transaction or not.<p>
* {@code null} means there haven't been any calls made on this transaction, and first operation will give this
* field actual value.
*/
- private Boolean sql;
+ private Boolean mvccOp;
/**
* Empty constructor required for {@link Externalizable}.
@@ -205,7 +206,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
- * @param ctx Cache registry.
+ * @param ctx Cache registry.
* @param implicit Implicit flag.
* @param implicitSingle Implicit with one key flag.
* @param sys System flag.
@@ -214,7 +215,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
* @param isolation Isolation.
* @param timeout Timeout.
* @param storeEnabled Store enabled flag.
- * @param sql Whether this transaction was started via SQL API or not, or {@code null} if unknown.
+ * @param mvccOp Whether this transaction was started via SQL API or not, or {@code null} if unknown.
* @param txSize Transaction size.
* @param subjId Subject ID.
* @param taskNameHash Task name hash code.
@@ -230,7 +231,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
TransactionIsolation isolation,
long timeout,
boolean storeEnabled,
- Boolean sql,
+ Boolean mvccOp,
int txSize,
@Nullable UUID subjId,
int taskNameHash,
@@ -257,7 +258,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
mappings = implicitSingle ? new IgniteTxMappingsSingleImpl() : new IgniteTxMappingsImpl();
- this.sql = sql;
+ this.mvccOp = mvccOp;
initResult();
@@ -574,6 +575,10 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
) {
assert key != null;
+ if (cacheCtx.mvccEnabled())
+ return mvccPutAllAsync0(cacheCtx, Collections.singletonMap(key, val),
+ entryProcessor == null ? null : Collections.singletonMap(key, entryProcessor), invokeArgs, retval, filter);
+
try {
beforePut(cacheCtx, retval, false);
@@ -628,7 +633,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (log.isDebugEnabled())
log.debug("Before acquiring transaction lock for put on key: " + enlisted);
- IgniteInternalFuture<Boolean>fut = cacheCtx.cache().txLockAsync(enlisted,
+ IgniteInternalFuture<Boolean> fut = cacheCtx.cache().txLockAsync(enlisted,
timeout,
this,
/*read*/entryProcessor != null, // Needed to force load from store.
@@ -696,6 +701,142 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
+ * Validate Tx mode.
+ *
+ * @param ctx Cache context.
+ * @throws IgniteCheckedException If tx mode is not supported.
+ */
+ protected void validateTxMode(GridCacheContext ctx) throws IgniteCheckedException {
+ if(!ctx.mvccEnabled() || pessimistic() && repeatableRead())
+ return;
+
+ throw new IgniteCheckedException("Only pessimistic repeatable read transactions are supported at the moment.");
+ }
+
+ /**
+ * Internal method for put and transform operations in Mvcc mode.
+ * Note: Only one of {@code map}, {@code transformMap} maps must be non-null.
+ *
+ * @param cacheCtx Context.
+ * @param map Key-value map to store.
+ * @param invokeMap Invoke map.
+ * @param invokeArgs Optional arguments for EntryProcessor.
+ * @param retval Key-transform value map to store.
+ * @param filter Filter.
+ * @return Operation future.
+ */
+ private <K, V> IgniteInternalFuture mvccPutAllAsync0(
+ final GridCacheContext cacheCtx,
+ @Nullable Map<? extends K, ? extends V> map,
+ @Nullable Map<? extends K, ? extends EntryProcessor<K, V, Object>> invokeMap,
+ @Nullable final Object[] invokeArgs,
+ final boolean retval,
+ @Nullable final CacheEntryPredicate filter
+ ) {
+ try {
+ validateTxMode(cacheCtx);
+
+ // TODO: IGNITE-9540: Fix invoke/invokeAll.
+ if(invokeMap != null)
+ MvccUtils.verifyMvccOperationSupport(cacheCtx, "invoke/invokeAll");
+
+ if (mvccSnapshot == null) {
+ MvccUtils.mvccTracker(cacheCtx, this);
+
+ assert mvccSnapshot != null;
+ }
+
+ beforePut(cacheCtx, retval, true);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture(e);
+ }
+
+ // Cached entry may be passed only from entry wrapper.
+ final Map<?, ?> map0 = map;
+ final Map<?, EntryProcessor<K, V, Object>> invokeMap0 = (Map<K, EntryProcessor<K, V, Object>>)invokeMap;
+
+ if (log.isDebugEnabled())
+ log.debug("Called putAllAsync(...) [tx=" + this + ", map=" + map0 + ", retval=" + retval + "]");
+
+ assert map0 != null || invokeMap0 != null;
+
+ if (F.isEmpty(map0) && F.isEmpty(invokeMap0)) {
+ if (implicit())
+ try {
+ commit();
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture<>(e);
+ }
+
+ return new GridFinishedFuture<>(new GridCacheReturn(true, false));
+ }
+
+ try {
+ // Set transform flag for transaction.
+ if (invokeMap != null)
+ transform = true;
+
+ Set<?> keys = map0 != null ? map0.keySet() : invokeMap0.keySet();
+
+ final Map<KeyCacheObject, CacheObject> enlisted = new HashMap<>(keys.size());
+
+ for (Object key : keys) {
+ if (isRollbackOnly())
+ return new GridFinishedFuture<>(timedOut() ? timeoutException() : rollbackException());
+
+ if (key == null) {
+ rollback();
+
+ throw new NullPointerException("Null key.");
+ }
+
+ Object val = map0 == null ? null : map0.get(key);
+ EntryProcessor entryProcessor = transform ? invokeMap.get(key) : null;
+
+ if (val == null && entryProcessor == null) {
+ setRollbackOnly();
+
+ throw new NullPointerException("Null value.");
+ }
+
+ KeyCacheObject cacheKey = cacheCtx.toCacheKeyObject(key);
+ CacheObject cacheVal = cacheCtx.toCacheObject(val);
+
+ enlisted.put(cacheKey, cacheVal);
+ }
+
+ return updateAsync(cacheCtx, new UpdateSourceIterator<IgniteBiTuple<KeyCacheObject, CacheObject>>() {
+
+ private Iterator<Map.Entry<KeyCacheObject, CacheObject>> it = enlisted.entrySet().iterator();
+
+ @Override public EnlistOperation operation() {
+ return EnlistOperation.UPSERT;
+ }
+
+ @Override public boolean hasNextX() throws IgniteCheckedException {
+ return it.hasNext();
+ }
+
+ @Override public IgniteBiTuple<KeyCacheObject, CacheObject> nextX() throws IgniteCheckedException {
+ Map.Entry<KeyCacheObject, CacheObject> next = it.next();
+
+ return new IgniteBiTuple<>(next.getKey(), next.getValue());
+ }
+ }, retval, filter, remainingTime(), true);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture(e);
+ }
+ catch (RuntimeException e) {
+ onException();
+
+ throw e;
+ }
+ }
+
+ /**
* Internal method for all put and transform operations. Only one of {@code map}, {@code transformMap}
* maps must be non-null.
*
@@ -717,6 +858,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
@Nullable Map<KeyCacheObject, GridCacheDrInfo> drMap,
final boolean retval
) {
+ if (cacheCtx.mvccEnabled())
+ return mvccPutAllAsync0(cacheCtx, map, invokeMap, invokeArgs, retval, null);
+
try {
beforePut(cacheCtx, retval, false);
}
@@ -1549,6 +1693,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
final boolean retval,
@Nullable final CacheEntryPredicate filter,
boolean singleRmv) {
+ if(cacheCtx.mvccEnabled())
+ return mvccRemoveAllAsync0(cacheCtx, keys, retval, filter);
+
try {
checkUpdatesAllowed(cacheCtx);
}
@@ -1558,9 +1705,6 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
cacheCtx.checkSecurity(SecurityPermission.CACHE_REMOVE);
- if (cacheCtx.mvccEnabled() && !isOperationAllowed(false))
- return txTypeMismatchFinishFuture();
-
if (retval)
needReturnValue(true);
@@ -1690,9 +1834,9 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
-1L);
PLC1<GridCacheReturn> plc1 = new PLC1<GridCacheReturn>(ret) {
+ /** {@inheritDoc} */
@Override protected GridCacheReturn postLock(GridCacheReturn ret)
- throws IgniteCheckedException
- {
+ throws IgniteCheckedException {
if (log.isDebugEnabled())
log.debug("Acquired transaction lock for remove on keys: " + enlisted);
@@ -1769,6 +1913,93 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
+ * Internal method for remove operations in Mvcc mode.
+ *
+ * @param cacheCtx Cache context.
+ * @param keys Keys to remove.
+ * @param retval Flag indicating whether a value should be returned.
+ * @param filter Filter.
+ * @return Future for asynchronous remove.
+ */
+ @SuppressWarnings("unchecked")
+ private <K, V> IgniteInternalFuture<GridCacheReturn> mvccRemoveAllAsync0(
+ final GridCacheContext cacheCtx,
+ @Nullable final Collection<? extends K> keys,
+ final boolean retval,
+ @Nullable final CacheEntryPredicate filter
+ ) {
+ try {
+ validateTxMode(cacheCtx);
+
+ if (mvccSnapshot == null) {
+ MvccUtils.mvccTracker(cacheCtx, this);
+
+ assert mvccSnapshot != null;
+ }
+
+ beforeRemove(cacheCtx, retval, true);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture(e);
+ }
+
+ if (F.isEmpty(keys)) {
+ if (implicit()) {
+ try {
+ commit();
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture<>(e);
+ }
+ }
+
+ return new GridFinishedFuture<>(new GridCacheReturn(localResult(), true));
+ }
+
+ init();
+
+ Set<KeyCacheObject> enlisted = new HashSet<>(keys.size());
+
+ try {
+ for (Object key : keys) {
+ if (isRollbackOnly())
+ return new GridFinishedFuture<>(timedOut() ? timeoutException() : rollbackException());
+
+ if (key == null) {
+ rollback();
+
+ throw new NullPointerException("Null key.");
+ }
+
+ KeyCacheObject cacheKey = cacheCtx.toCacheKeyObject(key);
+
+ enlisted.add(cacheKey);
+ }
+
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture(e);
+ }
+
+ return updateAsync(cacheCtx, new UpdateSourceIterator<KeyCacheObject>() {
+
+ private Iterator<KeyCacheObject> it = enlisted.iterator();
+
+ @Override public EnlistOperation operation() {
+ return EnlistOperation.DELETE;
+ }
+
+ @Override public boolean hasNextX() throws IgniteCheckedException {
+ return it.hasNext();
+ }
+
+ @Override public KeyCacheObject nextX() throws IgniteCheckedException {
+ return it.next();
+ }
+ }, retval, filter, remainingTime(), true);
+ }
+
+ /**
* @param cctx Cache context.
* @return Mvcc snapshot for read inside tx (initialized once for OPTIMISTIC SERIALIZABLE and REPEATABLE_READ txs).
*/
@@ -1846,10 +2077,67 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
+ * Executes key-value update operation in Mvcc mode.
+ *
+ * @param cacheCtx Cache context.
+ * @param it Entries iterator.
+ * @param retval Return value flag.
+ * @param filter Filter.
+ * @param timeout Timeout.
+ * @param sequential Sequential locking flag.
+ * @return Operation future.
+ */
+ private IgniteInternalFuture<GridCacheReturn> updateAsync(GridCacheContext cacheCtx,
+ UpdateSourceIterator<?> it,
+ boolean retval,
+ @Nullable CacheEntryPredicate filter,
+ long timeout,
+ boolean sequential) {
+ try {
+ final CacheOperationContext opCtx = cacheCtx.operationContextPerCall();
+
+ final boolean keepBinary = opCtx != null && opCtx.isKeepBinary();
+
+ /* TODO: IGNITE-9688: 'sequential' is always true here which can slowdown bulk operations,
+ but possibly we can safely optimize this. */
+
+ GridNearTxEnlistFuture fut = new GridNearTxEnlistFuture(cacheCtx, this,
+ timeout, it, 0, sequential, filter, retval);
+
+ fut.init();
+
+ return nonInterruptable(new GridEmbeddedFuture<>(fut.chain(new CX1<IgniteInternalFuture<GridCacheReturn>, Boolean>() {
+ @Override public Boolean applyx(IgniteInternalFuture<GridCacheReturn> fut0) throws IgniteCheckedException {
+ fut0.get();
+
+ return true;
+ }
+ }), new PLC1<GridCacheReturn>(null) {
+ @Override protected GridCacheReturn postLock(GridCacheReturn ret) throws IgniteCheckedException {
+ GridCacheReturn futRes = fut.get();
+
+ assert futRes != null;
+
+ mvccSnapshot.incrementOperationCounter();
+
+ return new GridCacheReturn(cacheCtx, true, keepBinary, futRes.value(), futRes.success());
+ }
+ }));
+ }
+ catch (RuntimeException e) {
+ onException();
+
+ throw e;
+ }
+ }
+
+ /**
+ * Executes update query operation in Mvcc mode.
+ *
* @param fut Enlist future.
* @return Operation future.
*/
- public IgniteInternalFuture<Long> updateAsync(GridNearTxAbstractEnlistFuture fut) {
+ public IgniteInternalFuture<Long> updateAsync(GridNearTxQueryAbstractEnlistFuture fut) {
fut.init();
return nonInterruptable(new GridEmbeddedFuture<>(fut.chain(new CX1<IgniteInternalFuture<Long>, Boolean>() {
@@ -1900,36 +2188,18 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (F.isEmpty(keys))
return new GridFinishedFuture<>(Collections.<K, V>emptyMap());
- if (cacheCtx.mvccEnabled() && !isOperationAllowed(false))
+ try {
+ validateTxMode(cacheCtx);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridFinishedFuture(e);
+ }
+
+ if (cacheCtx.mvccEnabled() && !isOperationAllowed(true))
return txTypeMismatchFinishFuture();
init();
- if (cacheCtx.mvccEnabled() && (optimistic() && !readCommitted()) && mvccTracker == null) {
- // TODO IGNITE-7388: support async tx rollback (e.g. on timeout).
- boolean canRemap = cctx.lockedTopologyVersion(null) == null;
-
- mvccTracker = new MvccQueryTrackerImpl(cacheCtx, canRemap);
-
- return new GridEmbeddedFuture<>(mvccTracker.requestSnapshot(topologyVersion()),
- new IgniteBiClosure<MvccSnapshot, Exception, IgniteInternalFuture<Map<K, V>>>() {
- @Override public IgniteInternalFuture<Map<K, V>> apply(MvccSnapshot snapshot, Exception e) {
- if (e != null)
- return new GridFinishedFuture<>(e);
-
- return getAllAsync(cacheCtx,
- entryTopVer,
- keys,
- deserializeBinary,
- skipVals,
- keepCacheObjects,
- skipStore,
- recovery,
- needVer);
- }
- });
- }
-
int keysCnt = keys.size();
boolean single = keysCnt == 1;
@@ -2234,8 +2504,8 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
* @param keepCacheObjects Keep cache objects flag.
* @param skipStore Skip store flag.
* @param recovery Recovery flag..
- * @throws IgniteCheckedException If failed.
* @return Enlisted keys.
+ * @throws IgniteCheckedException If failed.
*/
@SuppressWarnings({"RedundantTypeArguments"})
private <K, V> Collection<KeyCacheObject> enlistRead(
@@ -2568,8 +2838,8 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
GridInClosure3<KeyCacheObject, Object, GridCacheVersion> c =
new GridInClosure3<KeyCacheObject, Object, GridCacheVersion>() {
@Override public void apply(KeyCacheObject key,
- @Nullable Object val,
- @Nullable GridCacheVersion loadVer) {
+ @Nullable Object val,
+ @Nullable GridCacheVersion loadVer) {
if (log.isDebugEnabled())
log.debug("Loaded value from remote node [key=" + key + ", val=" + val + ']');
@@ -2729,7 +2999,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
- * @param cacheCtx Cache context.
+ * @param cacheCtx Cache context.
* @param readThrough Read through flag.
* @param async if {@code True}, then loading will happen in a separate thread.
* @param keys Keys.
@@ -2868,7 +3138,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
- * @param cacheCtx Cache context.
+ * @param cacheCtx Cache context.
* @param readThrough Read through flag.
* @param async if {@code True}, then loading will happen in a separate thread.
* @param keys Keys.
@@ -3498,7 +3768,10 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
if (log.isDebugEnabled())
log.debug("Committing near local tx: " + this);
- final NearTxFinishFuture fut, fut0 = finishFut; boolean fastFinish;
+ final NearTxFinishFuture fut;
+ final NearTxFinishFuture fut0 = finishFut;
+
+ boolean fastFinish;
if (fut0 != null || !FINISH_FUT_UPD.compareAndSet(this, null, fut = finishFuture(fastFinish = fastFinish(), true)))
return chainFinishFuture(finishFut, true, true, false);
@@ -3577,9 +3850,12 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
IgniteInternalFuture<?> prepFut = this.prepFut;
if (onTimeout && prepFut instanceof GridNearTxPrepareFutureAdapter && !prepFut.isDone())
- ((GridNearTxPrepareFutureAdapter) prepFut).onNearTxLocalTimeout();
+ ((GridNearTxPrepareFutureAdapter)prepFut).onNearTxLocalTimeout();
+
+ final NearTxFinishFuture fut;
+ final NearTxFinishFuture fut0 = finishFut;
- final NearTxFinishFuture fut, fut0 = finishFut; boolean fastFinish;
+ boolean fastFinish;
if (fut0 != null)
return chainFinishFuture(finishFut, false, clearThreadMap, onTimeout);
@@ -3627,9 +3903,11 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
}
/**
- * @return Transaction commit future.
+ * Finish transaction.
+ *
* @param fast {@code True} in case of fast finish.
* @param commit {@code True} if commit.
+ * @return Transaction commit future.
*/
private NearTxFinishFuture finishFuture(boolean fast, boolean commit) {
NearTxFinishFuture fut = fast ? new GridNearTxFastFinishFuture(this, commit) :
@@ -3724,7 +4002,7 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
* @return {@code True} if 'fast finish' path can be used for transaction completion.
*/
private boolean fastFinish() {
- return writeMap().isEmpty()
+ return writeMap().isEmpty() && !queryEnlisted()
&& ((optimistic() && !serializable()) || readMap().isEmpty())
&& (mappings.single() || F.view(mappings.mappings(), CU.FILTER_QUERY_MAPPING).isEmpty());
}
@@ -4174,14 +4452,14 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
* @return {@code true} if this transaction does not have type flag set or it matches invoking operation,
* {@code false} otherwise.
*/
- public boolean isOperationAllowed(boolean sqlOp) {
- if (sql == null) {
- sql = sqlOp;
+ public boolean isOperationAllowed(boolean mvccOp) {
+ if (this.mvccOp == null) {
+ this.mvccOp = mvccOp;
return true;
}
- return sql == sqlOp;
+ return this.mvccOp == mvccOp;
}
/**
@@ -4385,17 +4663,17 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
/**
* @param cacheCtx Cache context.
* @param retval Return value flag.
- * @param sql SQL operation flag.
+ * @param mvccOp SQL operation flag.
* @throws IgniteCheckedException If failed.
*/
- private void beforePut(GridCacheContext cacheCtx, boolean retval, boolean sql) throws IgniteCheckedException {
- assert !sql || cacheCtx.mvccEnabled();
+ private void beforePut(GridCacheContext cacheCtx, boolean retval, boolean mvccOp) throws IgniteCheckedException {
+ assert !mvccOp || cacheCtx.mvccEnabled();
checkUpdatesAllowed(cacheCtx);
cacheCtx.checkSecurity(SecurityPermission.CACHE_PUT);
- if (cacheCtx.mvccEnabled() && !isOperationAllowed(sql))
+ if (cacheCtx.mvccEnabled() && !isOperationAllowed(mvccOp))
throw new IgniteCheckedException(TX_TYPE_MISMATCH_ERR_MSG);
if (retval)
@@ -4408,6 +4686,28 @@ public class GridNearTxLocal extends GridDhtTxLocalAdapter implements GridTimeou
/**
* @param cacheCtx Cache context.
+ * @param retval Return value flag.
+ * @param mvccOp SQL operation flag.
+ * @throws IgniteCheckedException If failed.
+ */
+ private void beforeRemove(GridCacheContext cacheCtx, boolean retval, boolean mvccOp) throws IgniteCheckedException {
+ assert !mvccOp || cacheCtx.mvccEnabled();
+
+ checkUpdatesAllowed(cacheCtx);
+
+ cacheCtx.checkSecurity(SecurityPermission.CACHE_REMOVE);
+
+ if (cacheCtx.mvccEnabled() && !isOperationAllowed(mvccOp))
+ throw new IgniteCheckedException(TX_TYPE_MISMATCH_ERR_MSG);
+
+ if (retval)
+ needReturnValue(true);
+
+ checkValid();
+ }
+
+ /**
+ * @param cacheCtx Cache context.
* @throws IgniteCheckedException If updates are not allowed.
*/
private void checkUpdatesAllowed(GridCacheContext cacheCtx) throws IgniteCheckedException {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryAbstractEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryAbstractEnlistFuture.java
new file mode 100644
index 0000000..714c62d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryAbstractEnlistFuture.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+
+/**
+ *
+ */
+public abstract class GridNearTxQueryAbstractEnlistFuture extends GridNearTxAbstractEnlistFuture<Long> {
+ /**
+ * @param cctx Cache context.
+ * @param tx Transaction.
+ * @param timeout Timeout.
+ */
+ public GridNearTxQueryAbstractEnlistFuture(
+ GridCacheContext<?, ?> cctx, GridNearTxLocal tx, long timeout) {
+ super(cctx, tx, timeout, CU.longReducer());
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java
index 9a2dfa3..6d48b97 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistFuture.java
@@ -43,9 +43,7 @@ import static org.apache.ignite.internal.processors.cache.distributed.dht.NearTx
* Cache lock future.
*/
@SuppressWarnings("ForLoopReplaceableByForEach")
-public class GridNearTxQueryEnlistFuture extends GridNearTxAbstractEnlistFuture {
- /** */
- private static final long serialVersionUID = -2155104765461405820L;
+public class GridNearTxQueryEnlistFuture extends GridNearTxQueryAbstractEnlistFuture {
/** Involved cache ids. */
private final int[] cacheIds;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java
index dae1e81..d628de1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryEnlistResponse.java
@@ -38,7 +38,7 @@ import org.apache.ignite.plugin.extensions.communication.MessageWriter;
import org.jetbrains.annotations.Nullable;
/**
- *
+ * A response to {@link GridNearTxQueryEnlistRequest}.
*/
public class GridNearTxQueryEnlistResponse extends GridCacheIdMessage implements ExceptionAware {
/** */
@@ -99,6 +99,7 @@ public class GridNearTxQueryEnlistResponse extends GridCacheIdMessage implements
* @param lockVer Lock version.
* @param res Result.
* @param removeMapping Remove mapping flag.
+ * @param newDhtNodes New DHT nodes involved into transaction.
*/
public GridNearTxQueryEnlistResponse(int cacheId, IgniteUuid futId, int miniId, GridCacheVersion lockVer, long res,
boolean removeMapping, Set<UUID> newDhtNodes) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java
index 2452b92..b83339b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistFuture.java
@@ -62,10 +62,7 @@ import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_REA
* A future tracking requests for remote nodes transaction enlisting and locking
* of entries produced with complex DML queries requiring reduce step.
*/
-public class GridNearTxQueryResultsEnlistFuture extends GridNearTxAbstractEnlistFuture {
- /** */
- private static final long serialVersionUID = 4339957209840477447L;
-
+public class GridNearTxQueryResultsEnlistFuture extends GridNearTxQueryAbstractEnlistFuture {
/** */
public static final int DFLT_BATCH_SIZE = 1024;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java
index 94cacfa..48c63bc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxQueryResultsEnlistResponse.java
@@ -54,7 +54,7 @@ public class GridNearTxQueryResultsEnlistResponse extends GridNearTxQueryEnlistR
* @param res Result.
* @param dhtFutId Dht future id.
* @param dhtVer Dht version.
- * @param newDhtNodes New
+ * @param newDhtNodes New DHT nodes involved into transaction.
*/
public GridNearTxQueryResultsEnlistResponse(int cacheId,
IgniteUuid futId,
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
index ff1c85f..ca77bf9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccProcessorImpl.java
@@ -1233,7 +1233,8 @@ public class MvccProcessorImpl extends GridProcessorAdapter implements MvccProce
for (TxKey key : waitMap.keySet()) {
assert key.major() == snapshot.coordinatorVersion()
&& key.minor() > snapshot.cleanupVersion()
- || key.major() > snapshot.coordinatorVersion();
+ || key.major() > snapshot.coordinatorVersion() :
+ "key=" + key + ", snapshot=" + snapshot;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java
index f46d1e0..9a767ec 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/mvcc/MvccQueryTrackerImpl.java
@@ -27,6 +27,7 @@ import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteInClosure;
import org.jetbrains.annotations.NotNull;
@@ -46,7 +47,6 @@ public class MvccQueryTrackerImpl implements MvccQueryTracker {
private final IgniteLogger log;
/** */
- @GridToStringExclude
private long crdVer;
/** */
@@ -259,6 +259,9 @@ public class MvccQueryTrackerImpl implements MvccQueryTracker {
IgniteInternalFuture<AffinityTopologyVersion> waitFut =
cctx.shared().exchange().affinityReadyFuture(topVer.nextMinorVersion());
+ if(log.isDebugEnabled())
+ log.debug("Remap on new topology: " + waitFut);
+
if (waitFut == null)
requestSnapshot(cctx.shared().exchange().readyAffinityVersion(), lsnr);
else {
@@ -325,6 +328,11 @@ public class MvccQueryTrackerImpl implements MvccQueryTracker {
return true;
}
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(MvccQueryTrackerImpl.class, this);
+ }
+
/** */
private final class ListenerDecorator implements MvccSnapshotResponseListener {
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
index c57a790..16c30c9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheOffheapManager.java
@@ -49,6 +49,7 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdateNextSna
import org.apache.ignite.internal.pagemem.wal.record.delta.MetaPageUpdatePartitionDataRecord;
import org.apache.ignite.internal.pagemem.wal.record.delta.PartitionDestroyRecord;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
import org.apache.ignite.internal.processors.cache.CacheGroupContext;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -1795,13 +1796,15 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
GridCacheVersion ver,
long expireTime,
MvccSnapshot mvccVer,
+ CacheEntryPredicate filter,
boolean primary,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException {
+ boolean noCreate,
+ boolean retVal) throws IgniteCheckedException {
CacheDataStore delegate = init0(false);
- return delegate.mvccUpdate(
- cctx, key, val, ver, expireTime, mvccVer, primary, needHistory, noCreate);
+ return delegate.mvccUpdate(cctx, key, val, ver, expireTime, mvccVer, filter, primary,
+ needHistory, noCreate, retVal);
}
/** {@inheritDoc} */
@@ -1809,11 +1812,13 @@ public class GridCacheOffheapManager extends IgniteCacheOffheapManagerImpl imple
GridCacheContext cctx,
KeyCacheObject key,
MvccSnapshot mvccVer,
+ CacheEntryPredicate filter,
boolean primary,
- boolean needHistory) throws IgniteCheckedException {
+ boolean needHistory,
+ boolean retVal) throws IgniteCheckedException {
CacheDataStore delegate = init0(false);
- return delegate.mvccRemove(cctx, key, mvccVer, primary, needHistory);
+ return delegate.mvccRemove(cctx, key, mvccVer, filter, primary, needHistory, retVal);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
index fb6293c..d0e3dca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxLocalAdapter.java
@@ -1720,10 +1720,13 @@ public abstract class IgniteTxLocalAdapter extends IgniteTxAdapter implements Ig
*/
public void markQueryEnlisted(MvccSnapshot ver) {
if (!qryEnlisted) {
+ assert ver != null || mvccSnapshot != null;
+
if (mvccSnapshot == null)
mvccSnapshot = ver;
- cctx.coordinators().registerLocalTransaction(ver.coordinatorVersion(), ver.counter());
+ if(dht())
+ cctx.coordinators().registerLocalTransaction(mvccSnapshot.coordinatorVersion(), mvccSnapshot.counter());
qryEnlisted = true;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
index 7cc3e55..438c8ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxManager.java
@@ -466,7 +466,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
* @param concurrency Concurrency.
* @param isolation Isolation.
* @param timeout transaction timeout.
- * @param sql Whether this transaction is being started via SQL API or not, or {@code null} if unknown.
+ * @param mvccOp Whether this transaction is being started via SQL API or not, or {@code null} if unknown.
* @param txSize Expected transaction size.
* @param lb Label.
* @return New transaction.
@@ -479,7 +479,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
TransactionIsolation isolation,
long timeout,
boolean storeEnabled,
- Boolean sql,
+ Boolean mvccOp,
int txSize,
@Nullable String lb
) {
@@ -499,7 +499,7 @@ public class IgniteTxManager extends GridCacheSharedManagerAdapter {
isolation,
timeout,
storeEnabled,
- sql,
+ mvccOp,
txSize,
subjId,
taskNameHash,
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java
index 716094e..2a0b582 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java
@@ -21,9 +21,12 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheEntryEx;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.colocated.GridDhtDetachedCacheEntry;
import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils;
import org.apache.ignite.internal.processors.cache.mvcc.MvccVersion;
@@ -38,6 +41,8 @@ import org.apache.ignite.internal.processors.cache.tree.mvcc.search.MvccLinkAwar
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.MVCC_COUNTER_NA;
import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.MVCC_CRD_COUNTER_NA;
@@ -49,6 +54,7 @@ import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.isActiv
import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.isVisible;
import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.mvccVersionIsValid;
import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.unexpectedStateException;
+import static org.apache.ignite.internal.processors.cache.tree.mvcc.data.ResultType.FILTERED;
/**
*
@@ -94,6 +100,9 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
/** Whether tx has overridden it's own update. */
private static final int OWN_VALUE_OVERRIDDEN = DELETED << 1;
+ /** Force read full entry instead of header only. */
+ private static final int NEED_PREV_VALUE = OWN_VALUE_OVERRIDDEN << 1;
+
/** */
@GridToStringExclude
private final GridCacheContext cctx;
@@ -125,6 +134,10 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
/** */
private List<MvccLinkAwareSearchRow> historyRows;
+ /** */
+ @GridToStringExclude
+ private CacheEntryPredicate filter;
+
/**
* @param cctx Cache context.
* @param key Key.
@@ -148,10 +161,12 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
long expireTime,
MvccSnapshot mvccSnapshot,
MvccVersion newVer,
+ @Nullable CacheEntryPredicate filter,
boolean primary,
boolean lockOnly,
boolean needHistory,
- boolean fastUpdate) {
+ boolean fastUpdate,
+ boolean needPrevValue) {
super(key,
val,
ver,
@@ -163,6 +178,7 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
this.mvccSnapshot = mvccSnapshot;
this.cctx = cctx;
+ this.filter = filter;
this.keyAbsentBefore = primary; // True for primary and false for backup (backups do not use this flag).
assert !lockOnly || val == null;
@@ -181,6 +197,9 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
if (fastUpdate)
flags |= FAST_UPDATE;
+ if(needPrevValue)
+ flags |= NEED_PREV_VALUE;
+
setFlags(flags);
}
@@ -237,8 +256,18 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
if (removed)
setFlags(DELETED);
- else
- oldRow = row;
+ else {
+ // Actually, full row can be omitted for replace(k,newval) and putIfAbsent, but
+ // operation context is not available here and full row required if filter is set.
+ if (res == ResultType.PREV_NOT_NULL && (isFlagsSet(NEED_PREV_VALUE) || filter != null))
+ oldRow = tree.getRow(io, pageAddr, idx, RowData.FULL);
+ else
+ oldRow = row;
+ }
+
+ // TODO: IGNITE-9689: optimize filter usage here. See {@link org.apache.ignite.internal.processors.cache.CacheOperationFilter}.
+ if(filter != null && !applyFilter(res == ResultType.PREV_NOT_NULL ? oldRow.value() : null))
+ res = FILTERED;
setFlags(LAST_COMMITTED_FOUND | OWN_VALUE_OVERRIDDEN);
@@ -293,9 +322,14 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
else {
res = ResultType.PREV_NOT_NULL;
- oldRow = row;
-
keyAbsentBefore = false;
+
+ // Actually, full row can be omitted for replace(k,newval) and putIfAbsent, but
+ // operation context is not available here and full row required if filter is set.
+ if( (isFlagsSet(NEED_PREV_VALUE) || filter != null))
+ oldRow = tree.getRow(io, pageAddr, idx, RowData.FULL);
+ else
+ oldRow = row;
}
if (isFlagsSet(CHECK_VERSION)) {
@@ -337,9 +371,13 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
}
}
+ // TODO: IGNITE-9689: optimize filter usage here. See {@link org.apache.ignite.internal.processors.cache.CacheOperationFilter}.
+ if(filter != null && !applyFilter(res == ResultType.PREV_NOT_NULL ? oldRow.value() : null))
+ res = FILTERED;
+
// Lock entry for primary partition if needed.
// If invisible row is found for FAST_UPDATE case we should not lock row.
- if (isFlagsSet(PRIMARY | REMOVE_OR_LOCK) && !isFlagsSet(FAST_MISMATCH)) {
+ if (!isFlagsSet(DELETED) && isFlagsSet(PRIMARY | REMOVE_OR_LOCK) && !isFlagsSet(FAST_MISMATCH)) {
rowIo.setMvccLockCoordinatorVersion(pageAddr, idx, mvccCrd);
rowIo.setMvccLockCounter(pageAddr, idx, mvccCntr);
@@ -423,6 +461,22 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
return unsetFlags(FIRST);
}
+ /**
+ * Apply filter.
+ *
+ * @param val0 Previous value.
+ * @return Filter result.
+ */
+ private boolean applyFilter(final CacheObject val0) {
+ GridCacheEntryEx e = new GridDhtDetachedCacheEntry(cctx, key) {
+ @Nullable @Override public CacheObject peekVisibleValue() {
+ return val0;
+ }
+ };
+
+ return filter.apply(e);
+ }
+
/** {@inheritDoc} */
@Override public int state() {
return state;
@@ -436,10 +490,26 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult,
}
/**
- * @return {@code True} if previous value was non-null.
+ * @return Result type.
*/
- @Override public ResultType resultType() {
- return res == null ? ResultType.PREV_NULL : res;
+ @NotNull @Override public ResultType resultType() {
+ return res == null ? defaultResult() : res;
+ }
+
+ /**
+ * Evaluate default result type.
+ *
+ * @return Result type.
+ */
+ @NotNull private ResultType defaultResult() {
+ assert res == null;
+
+ if (filter != null && !applyFilter(null))
+ res = FILTERED;
+ else
+ res = ResultType.PREV_NULL; // Default.
+
+ return res;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/ResultType.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/ResultType.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/ResultType.java
index eecb4a5..16e7e1e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/ResultType.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/ResultType.java
@@ -30,5 +30,7 @@ public enum ResultType {
/** */
LOCKED,
/** */
- VERSION_MISMATCH
+ VERSION_MISMATCH,
+ /** */
+ FILTERED
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
index 9bbf03d..1a3c8d7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/GridCacheTestEntryEx.java
@@ -483,8 +483,8 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
/** {@inheritDoc} */
@Override public GridCacheUpdateTxResult mvccSet(@Nullable IgniteInternalTx tx, UUID affNodeId, CacheObject val,
long ttl0, AffinityTopologyVersion topVer, MvccSnapshot mvccVer,
- GridCacheOperation op, boolean needHistory,
- boolean noCreate) throws IgniteCheckedException, GridCacheEntryRemovedException {
+ GridCacheOperation op, boolean needHistory, boolean noCreate, CacheEntryPredicate filter, boolean retVal)
+ throws IgniteCheckedException, GridCacheEntryRemovedException {
rawPut(val, ttl);
return new GridCacheUpdateTxResult(true);
@@ -492,7 +492,8 @@ public class GridCacheTestEntryEx extends GridMetadataAwareAdapter implements Gr
/** {@inheritDoc} */
@Override public GridCacheUpdateTxResult mvccRemove(@Nullable IgniteInternalTx tx, UUID affNodeId,
- AffinityTopologyVersion topVer, MvccSnapshot mvccVer, boolean needHistory)
+ AffinityTopologyVersion topVer, MvccSnapshot mvccVer, boolean needHistory,
+ CacheEntryPredicate filter, boolean retVal)
throws IgniteCheckedException, GridCacheEntryRemovedException {
obsoleteVer = ver;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java
index dfc8b05..6a00ea4 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/IgniteCacheTxIteratorSelfTest.java
@@ -27,6 +27,9 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.NearCacheConfiguration;
import org.apache.ignite.configuration.TransactionConfiguration;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.IgniteCacheProxy;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.Transaction;
@@ -154,6 +157,13 @@ public class IgniteCacheTxIteratorSelfTest extends GridCommonAbstractTest {
for (TransactionIsolation iso : TransactionIsolation.values()) {
for (TransactionConcurrency con : TransactionConcurrency.values()) {
try (Transaction transaction = ignite.transactions().txStart(con, iso)) {
+ //TODO: IGNITE-7187: Fix when ticket will be implemented. (Near cache)
+ //TODO: IGNITE-7956: Fix when ticket will be implemented. (Eviction)
+ if (((IgniteCacheProxy)cache).context().mvccEnabled() &&
+ ((iso != TransactionIsolation.REPEATABLE_READ && con != TransactionConcurrency.PESSIMISTIC)
+ || nearEnabled || useEvicPlc))
+ return; // Nothing to do. Mode is not supported.
+
assertEquals(val, cache.get(key));
transaction.commit();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
index b2cbe05..c1718b5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractBasicCoordinatorFailoverTest.java
@@ -18,11 +18,11 @@
package org.apache.ignite.internal.processors.cache.mvcc;
import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.concurrent.Callable;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -52,10 +52,8 @@ import org.apache.ignite.transactions.TransactionIsolation;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
-import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE;
/**
* Base class for Mvcc coordinator failover test.
@@ -73,6 +71,8 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
ReadMode readMode,
WriteMode writeMode
) throws Exception {
+ assert concurrency == PESSIMISTIC && isolation == REPEATABLE_READ;
+
testSpi = true;
startGrids(3);
@@ -169,7 +169,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
final int KEYS = 100;
- final Map<Integer, Integer> vals = new HashMap<>();
+ final Map<Integer, Integer> vals = new LinkedHashMap<>();
for (int i = 0; i < KEYS; i++)
vals.put(i, 0);
@@ -298,7 +298,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
Integer val = 1;
while (!done.get()) {
- Map<Integer, Integer> vals = new HashMap<>();
+ Map<Integer, Integer> vals = new LinkedHashMap<>();
for (int i = 0; i < KEYS; i++)
vals.put(i, val);
@@ -479,9 +479,6 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
", crdChangeCnt=" + crdChangeCnt +
", readInTx=" + readInTx + ']');
- TransactionConcurrency concurrency = readMode == ReadMode.GET ? OPTIMISTIC : PESSIMISTIC; // TODO IGNITE-7184
- TransactionIsolation isolation = readMode == ReadMode.GET ? SERIALIZABLE : REPEATABLE_READ; // TODO IGNITE-7184
-
testSpi = true;
client = false;
@@ -510,7 +507,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
final IgniteCache cache = getNode.createCache(ccfg);
- final Set<Integer> keys = new HashSet<>();
+ final Set<Integer> keys = new TreeSet<>();
List<Integer> keys1 = primaryKeys(jcache(COORDS), 10);
List<Integer> keys2 = primaryKeys(jcache(COORDS + 1), 10);
@@ -518,7 +515,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
keys.addAll(keys1);
keys.addAll(keys2);
- Map<Integer, Integer> vals = new HashMap();
+ Map<Integer, Integer> vals = new LinkedHashMap();
for (Integer key : keys)
vals.put(key, -1);
@@ -544,7 +541,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
Map<Integer, Integer> res = null;
if (readInTx) {
- try (Transaction tx = getNode.transactions().txStart(concurrency, isolation)) {
+ try (Transaction tx = getNode.transactions().txStart(PESSIMISTIC, REPEATABLE_READ)) {
res = readAllByMode(cache, keys, readMode, INTEGER_CODEC);
tx.rollback();
@@ -581,7 +578,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
stopGrid(i);
for (int i = 0; i < 10; i++) {
- vals = new HashMap();
+ vals = new LinkedHashMap();
for (Integer key : keys)
vals.put(key, i);
@@ -636,7 +633,7 @@ public abstract class CacheMvccAbstractBasicCoordinatorFailoverTest extends Cach
final IgniteCache cache = client.createCache(ccfg);
- final Map<Integer, Integer> vals = new HashMap();
+ final Map<Integer, Integer> vals = new LinkedHashMap<>();
for (int i = 0; i < 100; i++)
vals.put(i, i);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractCoordinatorFailoverTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractCoordinatorFailoverTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractCoordinatorFailoverTest.java
index 54e4315..60f1a2f 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractCoordinatorFailoverTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractCoordinatorFailoverTest.java
@@ -113,27 +113,6 @@ public abstract class CacheMvccAbstractCoordinatorFailoverTest extends CacheMvcc
/**
* @throws Exception If failed.
*/
- public void testCoordinatorFailureSimpleSerializableTxPutGet() throws Exception {
- coordinatorFailureSimple(OPTIMISTIC, SERIALIZABLE, GET, PUT);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testCoordinatorFailureSimpleOptimisticTxPutGet() throws Exception {
- coordinatorFailureSimple(OPTIMISTIC, REPEATABLE_READ, GET, PUT);
- }
-
- /**
- * @throws Exception If failed.
- */
- public void testTxInProgressCoordinatorChangeSimple_ReadonlyPutGet() throws Exception {
- txInProgressCoordinatorChangeSimple(OPTIMISTIC, SERIALIZABLE, null, GET, PUT);
- }
-
- /**
- * @throws Exception If failed.
- */
public void testReadInProgressCoordinatorFailsSimple_FromClientPutGet() throws Exception {
readInProgressCoordinatorFailsSimple(true, null, GET, PUT);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
index fe450d1..6c6b8df 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractFeatureTest.java
@@ -173,7 +173,7 @@ public abstract class CacheMvccAbstractFeatureTest extends CacheMvccAbstractTest
int idx;
do {
- idx = (int) (Math.random() * 100) + 1;
+ idx = (int) (Math.random() * 100);
}
while (!keys.add(idx));
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
index a4962d1..c191849 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/mvcc/CacheMvccAbstractTest.java
@@ -22,10 +22,13 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -45,6 +48,7 @@ import org.apache.ignite.cache.CacheMode;
import org.apache.ignite.cache.CacheWriteSynchronizationMode;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.cache.query.FieldsQueryCursor;
+import org.apache.ignite.cache.query.QueryCursor;
import org.apache.ignite.cache.query.ScanQuery;
import org.apache.ignite.cache.query.SqlFieldsQuery;
import org.apache.ignite.cache.query.SqlQuery;
@@ -92,11 +96,9 @@ import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL_SNAPSHOT;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.cache.CacheMode.REPLICATED;
import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC;
-import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SCAN;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.ReadMode.SQL_SUM;
import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.DML;
-import static org.apache.ignite.internal.processors.cache.mvcc.CacheMvccAbstractTest.WriteMode.PUT;
import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
@@ -223,18 +225,21 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
/** {@inheritDoc} */
@Override protected void afterTest() throws Exception {
+ persistence = false;
+
try {
- verifyOldVersionsCleaned();
+ if(disableScheduledVacuum)
+ verifyOldVersionsCleaned();
verifyCoordinatorInternalState();
}
finally {
stopAllGrids();
- }
- MvccProcessorImpl.coordinatorAssignClosure(null);
+ MvccProcessorImpl.coordinatorAssignClosure(null);
- cleanPersistenceDir();
+ cleanPersistenceDir();
+ }
super.afterTest();
}
@@ -420,7 +425,7 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
Integer id1 = Math.min(i1, i2);
Integer id2 = Math.max(i1, i2);
- TreeSet<Integer> keys = new TreeSet<>();
+ Set<Integer> keys = new HashSet<>();
keys.add(id1);
keys.add(id2);
@@ -787,7 +792,7 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
* @param cache Cache.
* @return All accounts
*/
- private static Map<Integer, MvccTestAccount> getAllSql(TestCache<Integer, MvccTestAccount> cache) {
+ protected static Map<Integer, MvccTestAccount> getAllSql(TestCache<Integer, MvccTestAccount> cache) {
Map<Integer, MvccTestAccount> accounts = new HashMap<>();
SqlFieldsQuery qry = new SqlFieldsQuery("select _key, val, updateCnt from MvccTestAccount");
@@ -826,12 +831,28 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
* @param cache Cache.
* @param key Key.
*/
- private static void removeSql(TestCache<Integer, MvccTestAccount> cache, Integer key) {
+ protected static void removeSql(TestCache<Integer, MvccTestAccount> cache, Integer key) {
SqlFieldsQuery qry = new SqlFieldsQuery("delete from MvccTestAccount where _key=" + key);
cache.cache.query(qry).getAll();
}
+
+ /**
+ * Merge account by means of SQL API.
+ *
+ * @param cache Cache.
+ * @param key Key.
+ * @param val Value.
+ * @param updateCnt Update counter.
+ */
+ protected static void mergeSql(TestCache<Integer, MvccTestAccount> cache, Integer key, Integer val, Integer updateCnt) {
+ SqlFieldsQuery qry = new SqlFieldsQuery("merge into MvccTestAccount(_key, val, updateCnt) values " +
+ " (" + key+ ", " + val + ", " + updateCnt + ")");
+
+ cache.cache.query(qry).getAll();
+ }
+
/**
* Inserts account by means of SQL API.
*
@@ -867,9 +888,6 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
ReadMode readMode,
WriteMode writeMode
) throws Exception {
- if(readMode == SCAN && writeMode == PUT)
- fail("https://issues.apache.org/jira/browse/IGNITE-7764");
-
final int RANGE = 20;
final int writers = 4;
@@ -886,15 +904,23 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
info("Thread range [min=" + min + ", max=" + max + ']');
- Map<Integer, Integer> map = new HashMap<>();
+ // Sorted map for put to avoid deadlocks.
+ Map<Integer, Integer> map = new TreeMap<>();
+
+ // Unordered key sequence.
+ Set<Integer> keys = new LinkedHashSet<>();
int v = idx * 1_000_000;
boolean first = true;
while (!stop.get()) {
- while (map.size() < RANGE)
- map.put(rnd.nextInt(min, max), v);
+ while (keys.size() < RANGE) {
+ int key = rnd.nextInt(min, max);
+
+ if (keys.add(key))
+ map.put(key, v);
+ }
TestCache<Integer, Integer> cache = randomCache(caches, rnd);
@@ -903,9 +929,9 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
try (Transaction tx = txs.txStart(PESSIMISTIC, REPEATABLE_READ)) {
if (!first && rnd.nextBoolean()) {
- Map<Integer, Integer> res = readAllByMode(cache.cache, map.keySet(), readMode, INTEGER_CODEC);
+ Map<Integer, Integer> res = readAllByMode(cache.cache, keys, readMode, INTEGER_CODEC);
- for (Integer k : map.keySet())
+ for (Integer k : keys)
assertEquals("res=" + res, v - 1, (Object)res.get(k));
}
@@ -917,14 +943,12 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
}
if (rnd.nextBoolean()) {
- Map<Integer, Integer> res = readAllByMode(cache.cache, map.keySet(), readMode, INTEGER_CODEC);
+ Map<Integer, Integer> res = readAllByMode(cache.cache, keys, readMode, INTEGER_CODEC);
- for (Integer k : map.keySet())
+ for (Integer k : keys)
assertEquals("key=" + k, v, (Object)res.get(k));
}
- map.clear();
-
v++;
}
catch (Exception e) {
@@ -933,6 +957,8 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
finally {
cache.readUnlock();
+ keys.clear();
+
map.clear();
}
}
@@ -956,6 +982,8 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
int min = range * RANGE;
int max = min + RANGE;
+ keys.clear();
+
while (keys.size() < RANGE)
keys.add(rnd.nextInt(min, max));
@@ -1003,8 +1031,6 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
}
}
}
-
- keys.clear();
}
}
};
@@ -1054,9 +1080,6 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
)
throws Exception
{
- if(readMode == SCAN && writeMode == PUT)
- fail("https://issues.apache.org/jira/browse/IGNITE-7764");
-
final int TOTAL = 20;
assert N <= TOTAL;
@@ -1071,7 +1094,7 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
@Override public void apply(IgniteCache<Object, Object> cache) {
final IgniteTransactions txs = cache.unwrap(Ignite.class).transactions();
- Map<Integer, Integer> vals = new HashMap<>();
+ Map<Integer, Integer> vals = new LinkedHashMap<>();
for (int i = 0; i < TOTAL; i++)
vals.put(i, N);
@@ -1341,6 +1364,9 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
while (System.currentTimeMillis() < stopTime && !stop.get()) {
Thread.sleep(1000);
+ if (System.currentTimeMillis() >= stopTime || stop.get())
+ break;
+
if (restartMode != null) {
switch (restartMode) {
case RESTART_CRD: {
@@ -1806,12 +1832,15 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
}
});
+ Map res;
- Map res = (Map)cache.query(scanQry).getAll()
- .stream()
- .collect(Collectors.toMap(v -> ((IgniteBiTuple)v).getKey(), v -> ((IgniteBiTuple)v).getValue()));
+ try (QueryCursor qry = cache.query(scanQry)) {
+ res = (Map)qry.getAll()
+ .stream()
+ .collect(Collectors.toMap(v -> ((IgniteBiTuple)v).getKey(), v -> ((IgniteBiTuple)v).getValue()));
- assertTrue("res.size()=" + res.size() + ", keys.size()=" + keys.size(), res.size() <= keys.size());
+ assertTrue("res.size()=" + res.size() + ", keys.size()=" + keys.size(), res.size() <= keys.size());
+ }
return res;
@@ -1833,29 +1862,29 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
String qry = b.toString();
- SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(qry);
+ SqlFieldsQuery sqlFieldsQry = new SqlFieldsQuery(qry);
if (emulateLongQry)
sqlFieldsQry.setLazy(true).setPageSize(1);
List<List> rows;
- if (emulateLongQry) {
- FieldsQueryCursor<List> cur = cache.query(sqlFieldsQry);
-
- rows = new ArrayList<>();
+ try (FieldsQueryCursor<List> cur = cache.query(sqlFieldsQry)) {
+ if (emulateLongQry) {
+ rows = new ArrayList<>();
- for (List row : cur) {
- rows.add(row);
+ for (List row : cur) {
+ rows.add(row);
- doSleep(ThreadLocalRandom.current().nextInt(50));
+ doSleep(ThreadLocalRandom.current().nextInt(50));
+ }
}
+ else
+ rows = cur.getAll();
}
- else
- rows = cache.query(sqlFieldsQry).getAll();
if (rows.isEmpty())
- return Collections.EMPTY_MAP;
+ return Collections.emptyMap();
res = new HashMap();
@@ -1887,7 +1916,7 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
rows = cur.getAll();
if (rows.isEmpty())
- return Collections.EMPTY_MAP;
+ return Collections.emptyMap();
res = new HashMap();
@@ -2104,6 +2133,23 @@ public abstract class CacheMvccAbstractTest extends GridCommonAbstractTest {
}
/** {@inheritDoc} */
+ @Override public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ MvccTestAccount account = (MvccTestAccount)o;
+ return val == account.val &&
+ updateCnt == account.updateCnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override public int hashCode() {
+
+ return Objects.hash(val, updateCnt);
+ }
+
+ /** {@inheritDoc} */
@Override public String toString() {
return "MvccTestAccount{" +
"val=" + val +
[17/21] ignite git commit: IGNITE-9621: JDBC thin: remove warning
about unsupported transactions.
Posted by sb...@apache.org.
IGNITE-9621: JDBC thin: remove warning about unsupported transactions.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/a7480909
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/a7480909
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/a7480909
Branch: refs/heads/ignite-gg-14206
Commit: a7480909b50f6421e84877654e9f0f56c587cd2a
Parents: 37eedfc
Author: devozerov <vo...@gridgain.com>
Authored: Fri Sep 28 18:09:22 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 28 18:09:22 2018 +0300
----------------------------------------------------------------------
.../internal/jdbc/thin/JdbcThinConnection.java | 20 +++-----------------
1 file changed, 3 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/a7480909/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
index 547b1db..323a410 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/jdbc/thin/JdbcThinConnection.java
@@ -234,7 +234,7 @@ public class JdbcThinConnection implements Connection {
int resSetHoldability) throws SQLException {
ensureNotClosed();
- checkCursorOptions(resSetType, resSetConcurrency, resSetHoldability);
+ checkCursorOptions(resSetType, resSetConcurrency);
JdbcThinStatement stmt = new JdbcThinStatement(this, resSetHoldability, schema);
@@ -264,7 +264,7 @@ public class JdbcThinConnection implements Connection {
int resSetHoldability) throws SQLException {
ensureNotClosed();
- checkCursorOptions(resSetType, resSetConcurrency, resSetHoldability);
+ checkCursorOptions(resSetType, resSetConcurrency);
if (sql == null)
throw new SQLException("SQL string cannot be null.");
@@ -284,19 +284,14 @@ public class JdbcThinConnection implements Connection {
/**
* @param resSetType Cursor option.
* @param resSetConcurrency Cursor option.
- * @param resSetHoldability Cursor option.
* @throws SQLException If options unsupported.
*/
- private void checkCursorOptions(int resSetType, int resSetConcurrency,
- int resSetHoldability) throws SQLException {
+ private void checkCursorOptions(int resSetType, int resSetConcurrency) throws SQLException {
if (resSetType != TYPE_FORWARD_ONLY)
throw new SQLFeatureNotSupportedException("Invalid result set type (only forward is supported).");
if (resSetConcurrency != CONCUR_READ_ONLY)
throw new SQLFeatureNotSupportedException("Invalid concurrency (updates are not supported).");
-
- if (resSetHoldability != HOLD_CURSORS_OVER_COMMIT)
- LOG.warning("Transactions are not supported.");
}
/** {@inheritDoc} */
@@ -441,10 +436,6 @@ public class JdbcThinConnection implements Connection {
case Connection.TRANSACTION_READ_COMMITTED:
case Connection.TRANSACTION_REPEATABLE_READ:
case Connection.TRANSACTION_SERIALIZABLE:
- LOG.warning("Transactions are not supported.");
-
- break;
-
case Connection.TRANSACTION_NONE:
break;
@@ -459,8 +450,6 @@ public class JdbcThinConnection implements Connection {
@Override public int getTransactionIsolation() throws SQLException {
ensureNotClosed();
- LOG.warning("Transactions are not supported.");
-
return txIsolation;
}
@@ -494,9 +483,6 @@ public class JdbcThinConnection implements Connection {
@Override public void setHoldability(int holdability) throws SQLException {
ensureNotClosed();
- if (holdability != HOLD_CURSORS_OVER_COMMIT)
- LOG.warning("Transactions are not supported.");
-
if (holdability != HOLD_CURSORS_OVER_COMMIT && holdability != CLOSE_CURSORS_AT_COMMIT)
throw new SQLException("Invalid result set holdability value.");
[18/21] ignite git commit: IGNITE-9693 Scale up wal compression
workers to increase performance - Fixes #4831.
Posted by sb...@apache.org.
IGNITE-9693 Scale up wal compression workers to increase performance - Fixes #4831.
Signed-off-by: Ivan Rakov <ir...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/036bd074
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/036bd074
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/036bd074
Branch: refs/heads/ignite-gg-14206
Commit: 036bd074d8bfd25a2c4c463a60dde00604d11b9d
Parents: a748090
Author: Ivan Daschinskiy <iv...@gmail.com>
Authored: Fri Sep 28 18:03:45 2018 +0300
Committer: Ivan Rakov <ir...@apache.org>
Committed: Fri Sep 28 18:10:01 2018 +0300
----------------------------------------------------------------------
.../apache/ignite/IgniteSystemProperties.java | 5 +
.../pagemem/wal/IgniteWriteAheadLogManager.java | 3 +-
.../GridCacheDatabaseSharedManager.java | 11 +-
.../wal/FileWriteAheadLogManager.java | 208 ++++++++++++-------
.../wal/FsyncModeFileWriteAheadLogManager.java | 7 +-
.../persistence/wal/aware/SegmentAware.java | 28 ++-
.../wal/aware/SegmentCompressStorage.java | 80 +++++--
...PdsReserveWalSegmentsWithCompactionTest.java | 34 +++
.../persistence/pagemem/NoOpWALManager.java | 2 +-
.../persistence/wal/aware/SegmentAwareTest.java | 90 +++++---
.../ignite/testsuites/IgnitePdsTestSuite2.java | 2 +
11 files changed, 324 insertions(+), 146 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
index 5932de0..01fb02a 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteSystemProperties.java
@@ -901,6 +901,11 @@ public final class IgniteSystemProperties {
public static final String IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE = "IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE";
/**
+ * Count of WAL compressor worker threads. Default value is 4.
+ */
+ public static final String IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT = "IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT";
+
+ /**
* Whenever read load balancing is enabled, that means 'get' requests will be distributed between primary and backup
* nodes if it is possible and {@link CacheConfiguration#readFromBackup} is {@code true}.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
index 12fd3e9..4ffa347 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/IgniteWriteAheadLogManager.java
@@ -86,9 +86,8 @@ public interface IgniteWriteAheadLogManager extends GridCacheSharedManager, Igni
* Invoke this method to reserve WAL history since provided pointer and prevent it's deletion.
*
* @param start WAL pointer.
- * @throws IgniteException If failed to reserve.
*/
- public boolean reserve(WALPointer start) throws IgniteCheckedException;
+ public boolean reserve(WALPointer start);
/**
* Invoke this method to release WAL history since provided pointer that was previously reserved.
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
index 158c3b1..5e0b7cb 100755
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/GridCacheDatabaseSharedManager.java
@@ -1758,16 +1758,7 @@ public class GridCacheDatabaseSharedManager extends IgniteCacheDatabaseSharedMan
if (ptr == null)
return false;
- boolean reserved;
-
- try {
- reserved = cctx.wal().reserve(ptr);
- }
- catch (IgniteCheckedException e) {
- U.error(log, "Error while trying to reserve history", e);
-
- reserved = false;
- }
+ boolean reserved = cctx.wal().reserve(ptr);
if (reserved)
reservedForPreloading.put(new T2<>(grpId, partId), new T2<>(cntr, ptr));
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 5d165fd..43dfb8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -134,6 +134,7 @@ import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_CHECKPOINT_TRIGGER_ARCHIVE_SIZE_PERCENTAGE;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE;
+import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_MMAP;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SEGMENT_SYNC_TIMEOUT;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_WAL_SERIALIZER_VERSION;
@@ -257,6 +258,12 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private static final double THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE =
IgniteSystemProperties.getDouble(IGNITE_THRESHOLD_WAL_ARCHIVE_SIZE_PERCENTAGE, 0.5);
+ /**
+ * Number of WAL compressor worker threads.
+ */
+ private final int WAL_COMPRESSOR_WORKER_THREAD_CNT =
+ IgniteSystemProperties.getInteger(IGNITE_WAL_COMPRESSOR_WORKER_THREAD_CNT, 4);
+
/** */
private final boolean alwaysWriteFullPages;
@@ -415,7 +422,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
evt = ctx.event();
failureProcessor = ctx.failure();
- segmentAware = new SegmentAware(dsCfg.getWalSegments());
+ segmentAware = new SegmentAware(dsCfg.getWalSegments(), dsCfg.isWalCompactionEnabled());
}
/**
@@ -486,7 +493,8 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
segmentAware.setLastArchivedAbsoluteIndex(lastAbsArchivedIdx);
if (dsCfg.isWalCompactionEnabled()) {
- compressor = new FileCompressor();
+ if (compressor == null)
+ compressor = new FileCompressor(log);
if (decompressor == null) { // Preventing of two file-decompressor thread instantiations.
decompressor = new FileDecompressor(log);
@@ -895,7 +903,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/** {@inheritDoc} */
- @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+ @Override public boolean reserve(WALPointer start) {
assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
if (mode == WALMode.NONE)
@@ -1005,7 +1013,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
/** {@inheritDoc} */
@Override public void notchLastCheckpointPtr(WALPointer ptr) {
if (compressor != null)
- compressor.keepUncompressedIdxFrom(((FileWALPointer)ptr).index());
+ segmentAware.keepUncompressedIdxFrom(((FileWALPointer)ptr).index());
}
/** {@inheritDoc} */
@@ -1910,16 +1918,13 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* Responsible for compressing WAL archive segments.
* Also responsible for deleting raw copies of already compressed WAL archive segments if they are not reserved.
*/
- private class FileCompressor extends Thread {
- /** Current thread stopping advice. */
- private volatile boolean stopped;
-
- /** All segments prior to this (inclusive) can be compressed. */
- private volatile long minUncompressedIdxToKeep = -1L;
+ private class FileCompressor extends FileCompressorWorker {
+ /** Workers queue. */
+ List<FileCompressorWorker> workers = new ArrayList<>();
/** */
- FileCompressor() {
- super("wal-file-compressor%" + cctx.igniteInstanceName());
+ FileCompressor(IgniteLogger log) {
+ super(0, log);
}
/** */
@@ -1927,7 +1932,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
File[] toDel = walArchiveDir.listFiles(WAL_SEGMENT_TEMP_FILE_COMPACTED_FILTER);
for (File f : toDel) {
- if (stopped)
+ if (isCancelled())
return;
f.delete();
@@ -1936,82 +1941,118 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
FileDescriptor[] alreadyCompressed = scan(walArchiveDir.listFiles(WAL_SEGMENT_FILE_COMPACTED_FILTER));
if (alreadyCompressed.length > 0)
- segmentAware.lastCompressedIdx(alreadyCompressed[alreadyCompressed.length - 1].idx());
+ segmentAware.onSegmentCompressed(alreadyCompressed[alreadyCompressed.length - 1].idx());
+
+ for (int i = 1; i < calculateThreadCount(); i++) {
+ FileCompressorWorker worker = new FileCompressorWorker(i, log);
+
+ worker.start();
+
+ workers.add(worker);
+ }
}
/**
- * @param idx Minimum raw segment index that should be preserved from deletion.
+ * Calculate optimal additional compressor worker threads count. If quarter of proc threads greater
+ * than WAL_COMPRESSOR_WORKER_THREAD_CNT, use this value. Otherwise, reduce number of threads.
+ *
+ * @return Optimal number of compressor threads.
*/
- void keepUncompressedIdxFrom(long idx) {
- minUncompressedIdxToKeep = idx;
+ private int calculateThreadCount() {
+ int procNum = Runtime.getRuntime().availableProcessors();
+
+ // If quarter of proc threads greater than WAL_COMPRESSOR_WORKER_THREAD_CNT,
+ // use this value. Otherwise, reduce number of threads.
+ if (procNum >> 2 >= WAL_COMPRESSOR_WORKER_THREAD_CNT)
+ return WAL_COMPRESSOR_WORKER_THREAD_CNT;
+ else
+ return procNum >> 2;
}
- /**
- * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
- * Waits if there's no segment to archive right now.
- */
- private long tryReserveNextSegmentOrWait() throws IgniteCheckedException {
- long segmentToCompress = segmentAware.waitNextSegmentToCompress();
- boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0));
+ /** {@inheritDoc} */
+ @Override public void body() throws InterruptedException, IgniteInterruptedCheckedException{
+ init();
- return reserved ? segmentToCompress : -1;
+ super.body0();
}
/**
- * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
+ * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
*/
- private void deleteObsoleteRawSegments() {
- FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
+ private void shutdown() throws IgniteInterruptedCheckedException {
+ synchronized (this) {
+ for (FileCompressorWorker worker: workers)
+ U.cancel(worker);
- Set<Long> indices = new HashSet<>();
- Set<Long> duplicateIndices = new HashSet<>();
+ for (FileCompressorWorker worker: workers)
+ U.join(worker);
- for (FileDescriptor desc : descs) {
- if (!indices.add(desc.idx))
- duplicateIndices.add(desc.idx);
+ U.cancel(this);
}
- for (FileDescriptor desc : descs) {
- if (desc.isCompressed())
- continue;
+ U.join(this);
+ }
+ }
- // Do not delete reserved or locked segment and any segment after it.
- if (segmentReservedOrLocked(desc.idx))
- return;
+ /** */
+ private class FileCompressorWorker extends GridWorker {
+ /** */
+ private Thread thread;
- if (desc.idx < minUncompressedIdxToKeep && duplicateIndices.contains(desc.idx)) {
- if (!desc.file.delete())
- U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
- desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
- }
- }
+ /** */
+ FileCompressorWorker(int idx, IgniteLogger log) {
+ super(cctx.igniteInstanceName(), "wal-file-compressor-%" + cctx.igniteInstanceName() + "%-" + idx, log);
+ }
+
+ /** */
+ void start() {
+ thread = new IgniteThread(this);
+
+ thread.start();
+ }
+
+ /**
+ * Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
+ * Waits if there's no segment to archive right now.
+ */
+ private long tryReserveNextSegmentOrWait() throws IgniteInterruptedCheckedException{
+ long segmentToCompress = segmentAware.waitNextSegmentToCompress();
+
+ boolean reserved = reserve(new FileWALPointer(segmentToCompress, 0, 0));
+
+ return reserved ? segmentToCompress : -1;
}
/** {@inheritDoc} */
- @Override public void run() {
- init();
+ @Override protected void body() throws InterruptedException, IgniteInterruptedCheckedException {
+ body0();
+ }
- while (!Thread.currentThread().isInterrupted() && !stopped) {
- long currReservedSegment = -1;
+ /** */
+ private void body0() {
+ while (!isCancelled()) {
+ long segIdx = -1L;
try {
- deleteObsoleteRawSegments();
+ segIdx = tryReserveNextSegmentOrWait();
- currReservedSegment = tryReserveNextSegmentOrWait();
- if (currReservedSegment == -1)
+ if (segIdx <= segmentAware.lastCompressedIdx())
continue;
- File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment)
- + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
+ deleteObsoleteRawSegments();
+
+ File tmpZip = new File(walArchiveDir, FileDescriptor.fileName(segIdx)
+ + FilePageStoreManager.ZIP_SUFFIX + FilePageStoreManager.TMP_SUFFIX);
- File zip = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment) + FilePageStoreManager.ZIP_SUFFIX);
+ File zip = new File(walArchiveDir, FileDescriptor.fileName(segIdx) + FilePageStoreManager.ZIP_SUFFIX);
+
+ File raw = new File(walArchiveDir, FileDescriptor.fileName(segIdx));
- File raw = new File(walArchiveDir, FileDescriptor.fileName(currReservedSegment));
if (!Files.exists(raw.toPath()))
throw new IgniteCheckedException("WAL archive segment is missing: " + raw);
- compressSegmentToFile(currReservedSegment, raw, tmpZip);
+ compressSegmentToFile(segIdx, raw, tmpZip);
Files.move(tmpZip.toPath(), zip.toPath());
@@ -2022,27 +2063,27 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
if (evt.isRecordable(EVT_WAL_SEGMENT_COMPACTED)) {
evt.record(new WalSegmentCompactedEvent(
- cctx.discovery().localNode(),
- currReservedSegment,
- zip.getAbsoluteFile())
+ cctx.localNode(),
+ segIdx,
+ zip.getAbsoluteFile())
);
}
}
- segmentAware.lastCompressedIdx(currReservedSegment);
+ segmentAware.onSegmentCompressed(segIdx);
}
catch (IgniteInterruptedCheckedException ignore) {
Thread.currentThread().interrupt();
}
catch (IgniteCheckedException | IOException e) {
- U.error(log, "Compression of WAL segment [idx=" + currReservedSegment +
- "] was skipped due to unexpected error", e);
+ U.error(log, "Compression of WAL segment [idx=" + segIdx +
+ "] was skipped due to unexpected error", e);
- segmentAware.lastCompressedIdx(currReservedSegment);
+ segmentAware.onSegmentCompressed(segIdx);
}
finally {
- if (currReservedSegment != -1)
- release(new FileWALPointer(currReservedSegment, 0, 0));
+ if (segIdx != -1L)
+ release(new FileWALPointer(segIdx, 0, 0));
}
}
}
@@ -2053,7 +2094,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @param zip Zip file.
*/
private void compressSegmentToFile(long nextSegment, File raw, File zip)
- throws IOException, IgniteCheckedException {
+ throws IOException, IgniteCheckedException {
int segmentSerializerVer;
try (FileIO fileIO = ioFactory.create(raw)) {
@@ -2083,7 +2124,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
};
try (SingleSegmentLogicalRecordsIterator iter = new SingleSegmentLogicalRecordsIterator(
- log, cctx, ioFactory, BUF_SIZE, nextSegment, walArchiveDir, appendToZipC)) {
+ log, cctx, ioFactory, BUF_SIZE, nextSegment, walArchiveDir, appendToZipC)) {
while (iter.hasNextX())
iter.nextX();
@@ -2102,7 +2143,7 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
* @param ser Record Serializer.
*/
@NotNull private ByteBuffer prepareSwitchSegmentRecordBuffer(long nextSegment, RecordSerializer ser)
- throws IgniteCheckedException {
+ throws IgniteCheckedException {
SwitchSegmentRecord switchRecord = new SwitchSegmentRecord();
int switchRecordSize = ser.size(switchRecord);
@@ -2117,16 +2158,33 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
}
/**
- * @throws IgniteInterruptedCheckedException If failed to wait for thread shutdown.
+ * Deletes raw WAL segments if they aren't locked and already have compressed copies of themselves.
*/
- private void shutdown() throws IgniteInterruptedCheckedException {
- synchronized (this) {
- stopped = true;
+ private void deleteObsoleteRawSegments() {
+ FileDescriptor[] descs = scan(walArchiveDir.listFiles(WAL_SEGMENT_COMPACTED_OR_RAW_FILE_FILTER));
- notifyAll();
+ Set<Long> indices = new HashSet<>();
+ Set<Long> duplicateIndices = new HashSet<>();
+
+ for (FileDescriptor desc : descs) {
+ if (!indices.add(desc.idx))
+ duplicateIndices.add(desc.idx);
}
- U.join(this);
+ for (FileDescriptor desc : descs) {
+ if (desc.isCompressed())
+ continue;
+
+ // Do not delete reserved or locked segment and any segment after it.
+ if (segmentReservedOrLocked(desc.idx))
+ return;
+
+ if (desc.idx < segmentAware.keepUncompressedIdxFrom() && duplicateIndices.contains(desc.idx)) {
+ if (desc.file.exists() && !desc.file.delete())
+ U.warn(log, "Failed to remove obsolete WAL segment (make sure the process has enough rights): " +
+ desc.file.getAbsolutePath() + ", exists: " + desc.file.exists());
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
index df8f4de..1c0325e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FsyncModeFileWriteAheadLogManager.java
@@ -783,7 +783,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
}
/** {@inheritDoc} */
- @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+ @Override public boolean reserve(WALPointer start) {
assert start != null && start instanceof FileWALPointer : "Invalid start pointer: " + start;
if (mode == WALMode.NONE)
@@ -791,8 +791,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
FileArchiver archiver0 = archiver;
- if (archiver0 == null)
- throw new IgniteCheckedException("Could not reserve WAL segment: archiver == null");
+ assert archiver0 != null : "Could not reserve WAL segment: archiver == null";
archiver0.reserve(((FileWALPointer)start).index());
@@ -1912,7 +1911,7 @@ public class FsyncModeFileWriteAheadLogManager extends GridCacheSharedManagerAda
* Pessimistically tries to reserve segment for compression in order to avoid concurrent truncation.
* Waits if there's no segment to archive right now.
*/
- private long tryReserveNextSegmentOrWait() throws InterruptedException, IgniteCheckedException {
+ private long tryReserveNextSegmentOrWait() throws InterruptedException {
long segmentToCompress = lastCompressedIdx + 1;
synchronized (this) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
index 3379b74..6ba0399 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAware.java
@@ -36,15 +36,17 @@ public class SegmentAware {
/** Manages last archived index, emulates archivation in no-archiver mode. */
private final SegmentArchivedStorage segmentArchivedStorage = buildArchivedStorage(segmentLockStorage);
/** Storage of actual information about current index of compressed segments. */
- private final SegmentCompressStorage segmentCompressStorage = buildCompressStorage(segmentArchivedStorage);
+ private final SegmentCompressStorage segmentCompressStorage;
/** Storage of absolute current segment index. */
private final SegmentCurrentStateStorage segmentCurrStateStorage;
/**
* @param walSegmentsCnt Total WAL segments count.
+ * @param compactionEnabled Is wal compaction enabled.
*/
- public SegmentAware(int walSegmentsCnt) {
+ public SegmentAware(int walSegmentsCnt, boolean compactionEnabled) {
segmentCurrStateStorage = buildCurrentStateStorage(walSegmentsCnt, segmentArchivedStorage);
+ segmentCompressStorage = buildCompressStorage(segmentArchivedStorage, compactionEnabled);
}
/**
@@ -108,12 +110,12 @@ public class SegmentAware {
}
/**
- * Force set last compressed segment.
+ * Callback after segment compression finish.
*
- * @param lastCompressedIdx Segment which was last compressed.
+ * @param compressedIdx Index of compressed segment.
*/
- public void lastCompressedIdx(long lastCompressedIdx) {
- segmentCompressStorage.lastCompressedIdx(lastCompressedIdx);
+ public void onSegmentCompressed(long compressedIdx) {
+ segmentCompressStorage.onSegmentCompressed(compressedIdx);
}
/**
@@ -124,6 +126,20 @@ public class SegmentAware {
}
/**
+ * @param idx Minimum raw segment index that should be preserved from deletion.
+ */
+ public void keepUncompressedIdxFrom(long idx) {
+ segmentCompressStorage.keepUncompressedIdxFrom(idx);
+ }
+
+ /**
+ * @return Minimum raw segment index that should be preserved from deletion.
+ */
+ public long keepUncompressedIdxFrom() {
+ return segmentCompressStorage.keepUncompressedIdxFrom();
+ }
+
+ /**
* Update current WAL index.
*
* @param curAbsWalIdx New current WAL index.
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
index 30c9a2d..174fb46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentCompressStorage.java
@@ -18,6 +18,10 @@
package org.apache.ignite.internal.processors.cache.persistence.wal.aware;
import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Queue;
/**
* Storage of actual information about current index of compressed segments.
@@ -25,25 +29,50 @@ import org.apache.ignite.internal.IgniteInterruptedCheckedException;
public class SegmentCompressStorage {
/** Flag of interrupt waiting on this object. */
private volatile boolean interrupted;
+
/** Manages last archived index, emulates archivation in no-archiver mode. */
private final SegmentArchivedStorage segmentArchivedStorage;
+
+ /** If WAL compaction enabled. */
+ private final boolean compactionEnabled;
+
/** Last successfully compressed segment. */
private volatile long lastCompressedIdx = -1L;
+ /** Last enqueued to compress segment. */
+ private long lastEnqueuedToCompressIdx = -1L;
+
+ /** Segments to compress queue. */
+ private final Queue<Long> segmentsToCompress = new ArrayDeque<>();
+
+ /** List of currently compressing segments. */
+ private final List<Long> compressingSegments = new ArrayList<>();
+
+ /** Compressed segment with maximal index. */
+ private long lastMaxCompressedIdx = -1L;
+
+ /** Min uncompressed index to keep. */
+ private volatile long minUncompressedIdxToKeep = -1L;
+
/**
* @param segmentArchivedStorage Storage of last archived segment.
+ * @param compactionEnabled If WAL compaction enabled.
*/
- private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage) {
+ private SegmentCompressStorage(SegmentArchivedStorage segmentArchivedStorage, boolean compactionEnabled) {
this.segmentArchivedStorage = segmentArchivedStorage;
+ this.compactionEnabled = compactionEnabled;
+
this.segmentArchivedStorage.addObserver(this::onSegmentArchived);
}
/**
* @param segmentArchivedStorage Storage of last archived segment.
+ * @param compactionEnabled If WAL compaction enabled.
*/
- static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage) {
- SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage);
+ static SegmentCompressStorage buildCompressStorage(SegmentArchivedStorage segmentArchivedStorage,
+ boolean compactionEnabled) {
+ SegmentCompressStorage storage = new SegmentCompressStorage(segmentArchivedStorage, compactionEnabled);
segmentArchivedStorage.addObserver(storage::onSegmentArchived);
@@ -51,12 +80,20 @@ public class SegmentCompressStorage {
}
/**
- * Force set last compressed segment.
+ * Callback after segment compression finish.
*
- * @param lastCompressedIdx Segment which was last compressed.
+ * @param compressedIdx Index of compressed segment.
*/
- void lastCompressedIdx(long lastCompressedIdx) {
- this.lastCompressedIdx = lastCompressedIdx;
+ synchronized void onSegmentCompressed(long compressedIdx) {
+ if (compressedIdx > lastMaxCompressedIdx)
+ lastMaxCompressedIdx = compressedIdx;
+
+ compressingSegments.remove(compressedIdx);
+
+ if (!compressingSegments.isEmpty())
+ this.lastCompressedIdx = Math.min(lastMaxCompressedIdx, compressingSegments.get(0) - 1);
+ else
+ this.lastCompressedIdx = lastMaxCompressedIdx;
}
/**
@@ -71,13 +108,8 @@ public class SegmentCompressStorage {
* there's no segment to archive right now.
*/
synchronized long nextSegmentToCompressOrWait() throws IgniteInterruptedCheckedException {
- long segmentToCompress = lastCompressedIdx + 1;
-
try {
- while (
- segmentToCompress > segmentArchivedStorage.lastArchivedAbsoluteIndex()
- && !interrupted
- )
+ while (segmentsToCompress.peek() == null && !interrupted)
wait();
}
catch (InterruptedException e) {
@@ -86,7 +118,11 @@ public class SegmentCompressStorage {
checkInterrupted();
- return segmentToCompress;
+ Long idx = segmentsToCompress.poll();
+
+ compressingSegments.add(idx);
+
+ return idx == null ? -1L : idx;
}
/**
@@ -110,7 +146,23 @@ public class SegmentCompressStorage {
* Callback for waking up compressor when new segment is archived.
*/
private synchronized void onSegmentArchived(long lastAbsArchivedIdx) {
+ while (lastEnqueuedToCompressIdx < lastAbsArchivedIdx && compactionEnabled)
+ segmentsToCompress.add(++lastEnqueuedToCompressIdx);
+
notifyAll();
}
+ /**
+ * @param idx Minimum raw segment index that should be preserved from deletion.
+ */
+ void keepUncompressedIdxFrom(long idx) {
+ minUncompressedIdxToKeep = idx;
+ }
+
+ /**
+ * @return Minimum raw segment index that should be preserved from deletion.
+ */
+ long keepUncompressedIdxFrom() {
+ return minUncompressedIdxToKeep;
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java
new file mode 100644
index 0000000..bc34f29
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgnitePdsReserveWalSegmentsWithCompactionTest.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db;
+
+import org.apache.ignite.configuration.IgniteConfiguration;
+
+/**
+ *
+ */
+public class IgnitePdsReserveWalSegmentsWithCompactionTest extends IgnitePdsReserveWalSegmentsTest {
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(gridName);
+
+ cfg.getDataStorageConfiguration().setWalCompactionEnabled(true);
+
+ return cfg;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
index 811a231..df89419 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/NoOpWALManager.java
@@ -68,7 +68,7 @@ public class NoOpWALManager implements IgniteWriteAheadLogManager {
}
/** {@inheritDoc} */
- @Override public boolean reserve(WALPointer start) throws IgniteCheckedException {
+ @Override public boolean reserve(WALPointer start) {
return false;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
index 8287684..7840b09 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/aware/SegmentAwareTest.java
@@ -31,13 +31,12 @@ import static org.junit.Assert.assertThat;
* Test for {@link SegmentAware}.
*/
public class SegmentAwareTest extends TestCase {
-
/**
* Waiting finished when work segment is set.
*/
public void testFinishAwaitSegment_WhenExactWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
@@ -53,7 +52,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishAwaitSegment_WhenGreaterThanWaitingSegmentWasSet() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
@@ -69,7 +68,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishAwaitSegment_WhenNextSegmentEqualToWaitingOne() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
@@ -91,7 +90,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishAwaitSegment_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegment(5));
@@ -107,7 +106,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitSegmentForArchive_WhenWorkSegmentIncremented() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.curAbsWalIdx(5);
aware.setLastArchivedAbsoluteIndex(4);
@@ -126,7 +125,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitSegmentForArchive_WhenWorkSegmentGreaterValue() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.curAbsWalIdx(5);
aware.setLastArchivedAbsoluteIndex(4);
@@ -145,7 +144,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitSegmentForArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.curAbsWalIdx(5);
aware.setLastArchivedAbsoluteIndex(4);
@@ -164,7 +163,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testCorrectCalculateNextSegmentIndex() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.curAbsWalIdx(5);
@@ -180,7 +179,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitNextAbsoluteIndex_WhenMarkAsArchivedFirstSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(2);
+ SegmentAware aware = new SegmentAware(2, false);
aware.curAbsWalIdx(1);
aware.setLastArchivedAbsoluteIndex(-1);
@@ -199,7 +198,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitNextAbsoluteIndex_WhenSetToArchivedFirst() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(2);
+ SegmentAware aware = new SegmentAware(2, false);
aware.curAbsWalIdx(1);
aware.setLastArchivedAbsoluteIndex(-1);
@@ -218,7 +217,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitNextAbsoluteIndex_WhenOnlyForceInterruptWasCall() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(2);
+ SegmentAware aware = new SegmentAware(2, false);
aware.curAbsWalIdx(2);
aware.setLastArchivedAbsoluteIndex(-1);
@@ -243,7 +242,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishSegmentArchived_WhenSetExactWaitingSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
@@ -257,9 +256,9 @@ public class SegmentAwareTest extends TestCase {
/**
* Waiting finished when segment archived.
*/
- public void testFinishSegmentArchived_WhenMarkExactWatingSegment() throws IgniteCheckedException, InterruptedException {
+ public void testFinishSegmentArchived_WhenMarkExactWaitingSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
@@ -273,9 +272,9 @@ public class SegmentAwareTest extends TestCase {
/**
* Waiting finished when segment archived.
*/
- public void testFinishSegmentArchived_WhenSetGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException {
+ public void testFinishSegmentArchived_WhenSetGreaterThanWaitingSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
@@ -289,9 +288,9 @@ public class SegmentAwareTest extends TestCase {
/**
* Waiting finished when segment archived.
*/
- public void testFinishSegmentArchived_WhenMarkGreaterThanWatingSegment() throws IgniteCheckedException, InterruptedException {
+ public void testFinishSegmentArchived_WhenMarkGreaterThanWaitingSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
IgniteInternalFuture future = awaitThread(() -> aware.awaitSegmentArchived(5));
@@ -307,7 +306,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishSegmentArchived_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.curAbsWalIdx(5);
aware.setLastArchivedAbsoluteIndex(4);
@@ -326,7 +325,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testMarkAsMovedToArchive_WhenReleaseLockedSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.checkCanReadArchiveOrReserveWorkSegment(5);
@@ -344,7 +343,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testMarkAsMovedToArchive_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.checkCanReadArchiveOrReserveWorkSegment(5);
IgniteInternalFuture future = awaitThread(() -> aware.markAsMovedToArchive(5));
@@ -364,9 +363,9 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitSegmentToCompress_WhenSetLastArchivedSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, true);
- aware.lastCompressedIdx(5);
+ aware.onSegmentCompressed(5);
IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
@@ -382,9 +381,9 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitSegmentToCompress_WhenMarkLastArchivedSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, true);
- aware.lastCompressedIdx(5);
+ aware.onSegmentCompressed(5);
IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
@@ -400,9 +399,9 @@ public class SegmentAwareTest extends TestCase {
*/
public void testCorrectCalculateNextCompressSegment() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, true);
- aware.lastCompressedIdx(5);
+ aware.onSegmentCompressed(5);
aware.setLastArchivedAbsoluteIndex(6);
aware.lastTruncatedArchiveIdx(7);
@@ -418,8 +417,8 @@ public class SegmentAwareTest extends TestCase {
*/
public void testFinishWaitSegmentToCompress_WhenInterruptWasCall() throws IgniteCheckedException, InterruptedException {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
- aware.lastCompressedIdx(5);
+ SegmentAware aware = new SegmentAware(10, true);
+ aware.onSegmentCompressed(5);
IgniteInternalFuture future = awaitThread(aware::waitNextSegmentToCompress);
@@ -431,11 +430,34 @@ public class SegmentAwareTest extends TestCase {
}
/**
+ * Tests that {@link SegmentAware#onSegmentCompressed} returns segments in proper order.
+ */
+ public void testLastCompressedIdxProperOrdering() throws IgniteInterruptedCheckedException {
+ SegmentAware aware = new SegmentAware(10, true);
+
+ for (int i = 0; i < 5 ; i++) {
+ aware.setLastArchivedAbsoluteIndex(i);
+ aware.waitNextSegmentToCompress();
+ }
+
+ aware.onSegmentCompressed(0);
+
+ aware.onSegmentCompressed(4);
+ assertEquals(0, aware.lastCompressedIdx());
+ aware.onSegmentCompressed(1);
+ assertEquals(1, aware.lastCompressedIdx());
+ aware.onSegmentCompressed(3);
+ assertEquals(1, aware.lastCompressedIdx());
+ aware.onSegmentCompressed(2);
+ assertEquals(4, aware.lastCompressedIdx());
+ }
+
+ /**
* Segment reserve correctly.
*/
public void testReserveCorrectly() {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
//when: reserve one segment twice and one segment once.
aware.reserve(5);
@@ -478,7 +500,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testAssertFail_WhenReleaseUnreservedSegment() {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.reserve(5);
try {
@@ -497,7 +519,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testReserveWorkSegmentCorrectly() {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
//when: lock one segment twice.
aware.checkCanReadArchiveOrReserveWorkSegment(5);
@@ -530,7 +552,7 @@ public class SegmentAwareTest extends TestCase {
*/
public void testAssertFail_WhenReleaseUnreservedWorkSegment() {
//given: thread which awaited segment.
- SegmentAware aware = new SegmentAware(10);
+ SegmentAware aware = new SegmentAware(10, false);
aware.checkCanReadArchiveOrReserveWorkSegment(5);
try {
http://git-wip-us.apache.org/repos/asf/ignite/blob/036bd074/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
index a9f2601..7631834 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgnitePdsTestSuite2.java
@@ -34,6 +34,7 @@ import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOf
import org.apache.ignite.internal.processors.cache.persistence.baseline.IgniteOnlineNodeOutOfBaselineFullApiSelfTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsRebalancingOnNotStableTopologyTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsReserveWalSegmentsWithCompactionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.IgnitePdsWholeClusterRestartTest;
import org.apache.ignite.internal.processors.cache.persistence.db.SlowHistoricalRebalanceSmallHistoryTest;
import org.apache.ignite.internal.processors.cache.persistence.db.checkpoint.IgniteCheckpointDirtyPagesForLowLoadTest;
@@ -156,6 +157,7 @@ public class IgnitePdsTestSuite2 extends TestSuite {
suite.addTestSuite(IgnitePdsExchangeDuringCheckpointTest.class);
suite.addTestSuite(IgnitePdsReserveWalSegmentsTest.class);
+ suite.addTestSuite(IgnitePdsReserveWalSegmentsWithCompactionTest.class);
// new style folders with generated consistent ID test
suite.addTestSuite(IgniteUidAsConsistentIdMigrationTest.class);
[04/21] ignite git commit: IGNITE-9711: [ML] Remove IgniteThread
wrapper from ml examples
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
index e99494b..0c8b562 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
@@ -33,7 +33,6 @@ import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
-import org.apache.ignite.thread.IgniteThread;
/**
* Change classification algorithm that was used in {@link Step_5_Scaling} from decision tree to kNN
@@ -55,80 +54,74 @@ public class Step_6_KNN {
System.out.println(">>> Tutorial step 6 (kNN) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_6_KNN.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- featureExtractor
- );
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
- .withP(1)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- KNNClassificationTrainer trainer = new KNNClassificationTrainer();
-
- // Train decision tree model.
- NNClassificationModel mdl = trainer.fit(
- ignite,
- dataCache,
- normalizationPreprocessor,
- lbExtractor
- ).withK(1).withStrategy(NNStrategy.WEIGHTED);
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
- System.out.println("\n>>> Trained model: " + mdl);
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- double accuracy = Evaluator.evaluate(
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here.
+ .fit(ignite,
dataCache,
- mdl,
- normalizationPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ featureExtractor
+ );
+
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
+ dataCache,
+ strEncoderPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
+ IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
+ .fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor
+ );
- System.out.println(">>> Tutorial step 6 (kNN) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
+ .withP(1)
+ .fit(
+ ignite,
+ dataCache,
+ minMaxScalerPreprocessor
+ );
- igniteThread.start();
- igniteThread.join();
+ KNNClassificationTrainer trainer = new KNNClassificationTrainer();
+
+ // Train decision tree model.
+ NNClassificationModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ normalizationPreprocessor,
+ lbExtractor
+ ).withK(1).withStrategy(NNStrategy.WEIGHTED);
+
+ System.out.println("\n>>> Trained model: " + mdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ normalizationPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 6 (kNN) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
index 2ce2b27..c6d033c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
@@ -34,7 +34,6 @@ import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* The highest accuracy in the previous example ({@link Step_6_KNN}) is the result of
@@ -57,86 +56,79 @@ public class Step_7_Split_train_test {
System.out.println(">>> Tutorial step 7 (split to train and test) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_7_Split_train_test.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
- .split(0.75);
-
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- featureExtractor
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
+
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+
+ TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
+ .split(0.75);
+
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here.
+ .fit(ignite,
+ dataCache,
+ featureExtractor
+ );
+
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
+ dataCache,
+ strEncoderPreprocessor
);
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
- .withP(1)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
-
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
+ IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
+ .fit(
ignite,
dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor
+ imputingPreprocessor
);
- System.out.println("\n>>> Trained model: " + mdl);
-
- double accuracy = Evaluator.evaluate(
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
+ .withP(1)
+ .fit(
+ ignite,
dataCache,
- split.getTestFilter(),
- mdl,
- normalizationPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ minMaxScalerPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 7 (split to train and test) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + mdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ split.getTestFilter(),
+ mdl,
+ normalizationPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 7 (split to train and test) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
index 83c2cca..d83e14a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
@@ -36,7 +36,6 @@ import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* To choose the best hyperparameters the cross-validation will be used in this example.
@@ -69,137 +68,130 @@ public class Step_8_CV {
System.out.println(">>> Tutorial step 8 (cross-validation) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_8_CV.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
- .split(0.75);
-
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- featureExtractor
- );
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
- IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- // Tune hyperparams with K-fold Cross-Validation on the split training set.
- int[] pSet = new int[]{1, 2};
- int[] maxDeepSet = new int[]{1, 2, 3, 4, 5, 10, 20};
- int bestP = 1;
- int bestMaxDeep = 1;
- double avg = Double.MIN_VALUE;
-
- for(int p: pSet){
- for(int maxDeep: maxDeepSet){
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor
- = new NormalizationTrainer<Integer, Object[]>()
- .withP(p)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- DecisionTreeClassificationTrainer trainer
- = new DecisionTreeClassificationTrainer(maxDeep, 0);
-
- CrossValidation<DecisionTreeNode, Double, Integer, Object[]> scoreCalculator
- = new CrossValidation<>();
-
- double[] scores = scoreCalculator.score(
- trainer,
- new Accuracy<>(),
- ignite,
- dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor,
- 3
- );
+ TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
+ .split(0.75);
- System.out.println("Scores are: " + Arrays.toString(scores));
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here.
+ .fit(ignite,
+ dataCache,
+ featureExtractor
+ );
- final double currAvg = Arrays.stream(scores).average().orElse(Double.MIN_VALUE);
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
+ dataCache,
+ strEncoderPreprocessor
+ );
- if(currAvg > avg) {
- avg = currAvg;
- bestP = p;
- bestMaxDeep = maxDeep;
- }
+ IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
+ .fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor
+ );
- System.out.println("Avg is: " + currAvg + " with p: " + p + " with maxDeep: " + maxDeep);
- }
- }
+ // Tune hyperparams with K-fold Cross-Validation on the split training set.
+ int[] pSet = new int[]{1, 2};
+ int[] maxDeepSet = new int[]{1, 2, 3, 4, 5, 10, 20};
+ int bestP = 1;
+ int bestMaxDeep = 1;
+ double avg = Double.MIN_VALUE;
+
+ for(int p: pSet){
+ for(int maxDeep: maxDeepSet){
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor
+ = new NormalizationTrainer<Integer, Object[]>()
+ .withP(p)
+ .fit(
+ ignite,
+ dataCache,
+ minMaxScalerPreprocessor
+ );
- System.out.println("Train with p: " + bestP + " and maxDeep: " + bestMaxDeep);
+ DecisionTreeClassificationTrainer trainer
+ = new DecisionTreeClassificationTrainer(maxDeep, 0);
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
- .withP(bestP)
- .fit(
+ CrossValidation<DecisionTreeNode, Double, Integer, Object[]> scoreCalculator
+ = new CrossValidation<>();
+
+ double[] scores = scoreCalculator.score(
+ trainer,
+ new Accuracy<>(),
ignite,
dataCache,
- minMaxScalerPreprocessor
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor,
+ 3
);
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(bestMaxDeep, 0);
-
- // Train decision tree model.
- DecisionTreeNode bestMdl = trainer.fit(
- ignite,
- dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor
- );
-
- System.out.println("\n>>> Trained model: " + bestMdl);
+ System.out.println("Scores are: " + Arrays.toString(scores));
- double accuracy = Evaluator.evaluate(
- dataCache,
- split.getTestFilter(),
- bestMdl,
- normalizationPreprocessor,
- lbExtractor,
- new Accuracy<>()
- );
+ final double currAvg = Arrays.stream(scores).average().orElse(Double.MIN_VALUE);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
+ if(currAvg > avg) {
+ avg = currAvg;
+ bestP = p;
+ bestMaxDeep = maxDeep;
+ }
- System.out.println(">>> Tutorial step 8 (cross-validation) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
+ System.out.println("Avg is: " + currAvg + " with p: " + p + " with maxDeep: " + maxDeep);
+ }
}
- });
- igniteThread.start();
+ System.out.println("Train with p: " + bestP + " and maxDeep: " + bestMaxDeep);
+
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
+ .withP(bestP)
+ .fit(
+ ignite,
+ dataCache,
+ minMaxScalerPreprocessor
+ );
- igniteThread.join();
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(bestMaxDeep, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode bestMdl = trainer.fit(
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + bestMdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ split.getTestFilter(),
+ bestMdl,
+ normalizationPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 8 (cross-validation) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
index 73a0303..594c0eb 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
@@ -38,7 +38,6 @@ import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* To choose the best hyperparameters the cross-validation with {@link ParamGrid} will be used in this example.
@@ -71,122 +70,115 @@ public class Step_8_CV_with_Param_Grid {
System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_8_CV_with_Param_Grid.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare" .
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
- .split(0.75);
-
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- featureExtractor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
- .withP(2)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- // Tune hyperparams with K-fold Cross-Validation on the split training set.
-
- DecisionTreeClassificationTrainer trainerCV = new DecisionTreeClassificationTrainer();
-
- CrossValidation<DecisionTreeNode, Double, Integer, Object[]> scoreCalculator
- = new CrossValidation<>();
-
- ParamGrid paramGrid = new ParamGrid()
- .addHyperParam("maxDeep", new Double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 10.0})
- .addHyperParam("minImpurityDecrease", new Double[]{0.0, 0.25, 0.5});
-
- CrossValidationResult crossValidationRes = scoreCalculator.score(
- trainerCV,
- new Accuracy<>(),
- ignite,
- dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor,
- 3,
- paramGrid
- );
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- System.out.println("Train with maxDeep: " + crossValidationRes.getBest("maxDeep")
- + " and minImpurityDecrease: " + crossValidationRes.getBest("minImpurityDecrease"));
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare" .
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer()
- .withMaxDeep(crossValidationRes.getBest("maxDeep"))
- .withMinImpurityDecrease(crossValidationRes.getBest("minImpurityDecrease"));
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- System.out.println(crossValidationRes);
+ TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
+ .split(0.75);
- System.out.println("Best score: " + Arrays.toString(crossValidationRes.getBestScore()));
- System.out.println("Best hyper params: " + crossValidationRes.getBestHyperParams());
- System.out.println("Best average score: " + crossValidationRes.getBestAvgScore());
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here.
+ .fit(ignite,
+ dataCache,
+ featureExtractor
+ );
- crossValidationRes.getScoringBoard().forEach((hyperParams, score)
- -> System.out.println("Score " + Arrays.toString(score) + " for hyper params " + hyperParams));
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
+ dataCache,
+ strEncoderPreprocessor
+ );
- // Train decision tree model.
- DecisionTreeNode bestMdl = trainer.fit(
+ IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
+ .fit(
ignite,
dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor
+ imputingPreprocessor
);
- System.out.println("\n>>> Trained model: " + bestMdl);
-
- double accuracy = Evaluator.evaluate(
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
+ .withP(2)
+ .fit(
+ ignite,
dataCache,
- split.getTestFilter(),
- bestMdl,
- normalizationPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ minMaxScalerPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ // Tune hyperparams with K-fold Cross-Validation on the split training set.
+
+ DecisionTreeClassificationTrainer trainerCV = new DecisionTreeClassificationTrainer();
+
+ CrossValidation<DecisionTreeNode, Double, Integer, Object[]> scoreCalculator
+ = new CrossValidation<>();
+
+ ParamGrid paramGrid = new ParamGrid()
+ .addHyperParam("maxDeep", new Double[]{1.0, 2.0, 3.0, 4.0, 5.0, 10.0, 10.0})
+ .addHyperParam("minImpurityDecrease", new Double[]{0.0, 0.25, 0.5});
+
+ CrossValidationResult crossValidationRes = scoreCalculator.score(
+ trainerCV,
+ new Accuracy<>(),
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor,
+ 3,
+ paramGrid
+ );
+
+ System.out.println("Train with maxDeep: " + crossValidationRes.getBest("maxDeep")
+ + " and minImpurityDecrease: " + crossValidationRes.getBest("minImpurityDecrease"));
+
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer()
+ .withMaxDeep(crossValidationRes.getBest("maxDeep"))
+ .withMinImpurityDecrease(crossValidationRes.getBest("minImpurityDecrease"));
+
+ System.out.println(crossValidationRes);
+
+ System.out.println("Best score: " + Arrays.toString(crossValidationRes.getBestScore()));
+ System.out.println("Best hyper params: " + crossValidationRes.getBestHyperParams());
+ System.out.println("Best average score: " + crossValidationRes.getBestAvgScore());
+
+ crossValidationRes.getScoringBoard().forEach((hyperParams, score)
+ -> System.out.println("Score " + Arrays.toString(score) + " for hyper params " + hyperParams));
+
+ // Train decision tree model.
+ DecisionTreeNode bestMdl = trainer.fit(
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + bestMdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ split.getTestFilter(),
+ bestMdl,
+ normalizationPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
index 088caf7..4e1e005 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
@@ -39,7 +39,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
-import org.apache.ignite.thread.IgniteThread;
/**
* Change classification algorithm that was used in {@link Step_8_CV_with_Param_Grid} from decision tree to logistic
@@ -62,173 +61,166 @@ public class Step_9_Go_to_LogReg {
System.out.println(">>> Tutorial step 9 (logistic regression) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_9_Go_to_LogReg.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare"
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
- .split(0.75);
-
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here
- .fit(ignite,
- dataCache,
- featureExtractor
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare"
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
+
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+
+ TrainTestSplit<Integer, Object[]> split = new TrainTestDatasetSplitter<Integer, Object[]>()
+ .split(0.75);
+
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here
+ .fit(ignite,
+ dataCache,
+ featureExtractor
+ );
+
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
+ dataCache,
+ strEncoderPreprocessor
);
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
-
- // Tune hyperparams with K-fold Cross-Validation on the split training set.
- int[] pSet = new int[]{1, 2};
- int[] maxIterationsSet = new int[]{ 100, 1000};
- int[] batchSizeSet = new int[]{100, 10};
- int[] locIterationsSet = new int[]{10, 100};
- double[] learningRateSet = new double[]{0.1, 0.2, 0.5};
-
- int bestP = 1;
- int bestMaxIterations = 100;
- int bestBatchSize = 10;
- int bestLocIterations = 10;
- double bestLearningRate = 0.0;
- double avg = Double.MIN_VALUE;
-
- for(int p: pSet){
- for(int maxIterations: maxIterationsSet) {
- for (int batchSize : batchSizeSet) {
- for (int locIterations : locIterationsSet) {
- for (double learningRate : learningRateSet) {
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor
- = new NormalizationTrainer<Integer, Object[]>()
- .withP(p)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- LogisticRegressionSGDTrainer<?> trainer
- = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(learningRate),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), maxIterations, batchSize, locIterations, 123L);
-
- CrossValidation<LogisticRegressionModel, Double, Integer, Object[]>
- scoreCalculator = new CrossValidation<>();
-
- double[] scores = scoreCalculator.score(
- trainer,
- new Accuracy<>(),
+ IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
+ .fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor
+ );
+
+ // Tune hyperparams with K-fold Cross-Validation on the split training set.
+ int[] pSet = new int[]{1, 2};
+ int[] maxIterationsSet = new int[]{ 100, 1000};
+ int[] batchSizeSet = new int[]{100, 10};
+ int[] locIterationsSet = new int[]{10, 100};
+ double[] learningRateSet = new double[]{0.1, 0.2, 0.5};
+
+ int bestP = 1;
+ int bestMaxIterations = 100;
+ int bestBatchSize = 10;
+ int bestLocIterations = 10;
+ double bestLearningRate = 0.0;
+ double avg = Double.MIN_VALUE;
+
+ for(int p: pSet){
+ for(int maxIterations: maxIterationsSet) {
+ for (int batchSize : batchSizeSet) {
+ for (int locIterations : locIterationsSet) {
+ for (double learningRate : learningRateSet) {
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor
+ = new NormalizationTrainer<Integer, Object[]>()
+ .withP(p)
+ .fit(
ignite,
dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor,
- 3
+ minMaxScalerPreprocessor
);
- System.out.println("Scores are: " + Arrays.toString(scores));
-
- final double currAvg = Arrays.stream(scores).average().orElse(Double.MIN_VALUE);
-
- if (currAvg > avg) {
- avg = currAvg;
- bestP = p;
- bestMaxIterations = maxIterations;
- bestBatchSize = batchSize;
- bestLearningRate = learningRate;
- bestLocIterations = locIterations;
- }
-
- System.out.println("Avg is: " + currAvg
- + " with p: " + p
- + " with maxIterations: " + maxIterations
- + " with batchSize: " + batchSize
- + " with learningRate: " + learningRate
- + " with locIterations: " + locIterations
- );
+ LogisticRegressionSGDTrainer<?> trainer
+ = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(learningRate),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ), maxIterations, batchSize, locIterations, 123L);
+
+ CrossValidation<LogisticRegressionModel, Double, Integer, Object[]>
+ scoreCalculator = new CrossValidation<>();
+
+ double[] scores = scoreCalculator.score(
+ trainer,
+ new Accuracy<>(),
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor,
+ 3
+ );
+
+ System.out.println("Scores are: " + Arrays.toString(scores));
+
+ final double currAvg = Arrays.stream(scores).average().orElse(Double.MIN_VALUE);
+
+ if (currAvg > avg) {
+ avg = currAvg;
+ bestP = p;
+ bestMaxIterations = maxIterations;
+ bestBatchSize = batchSize;
+ bestLearningRate = learningRate;
+ bestLocIterations = locIterations;
}
+
+ System.out.println("Avg is: " + currAvg
+ + " with p: " + p
+ + " with maxIterations: " + maxIterations
+ + " with batchSize: " + batchSize
+ + " with learningRate: " + learningRate
+ + " with locIterations: " + locIterations
+ );
}
}
}
}
+ }
- System.out.println("Train "
- + " with p: " + bestP
- + " with maxIterations: " + bestMaxIterations
- + " with batchSize: " + bestBatchSize
- + " with learningRate: " + bestLearningRate
- + " with locIterations: " + bestLocIterations
- );
-
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
- .withP(bestP)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(bestLearningRate),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), bestMaxIterations, bestBatchSize, bestLocIterations, 123L);
-
- System.out.println(">>> Perform the training to get the model.");
- LogisticRegressionModel bestMdl = trainer.fit(
+ System.out.println("Train "
+ + " with p: " + bestP
+ + " with maxIterations: " + bestMaxIterations
+ + " with batchSize: " + bestBatchSize
+ + " with learningRate: " + bestLearningRate
+ + " with locIterations: " + bestLocIterations
+ );
+
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
+ .withP(bestP)
+ .fit(
ignite,
dataCache,
- split.getTrainFilter(),
- normalizationPreprocessor,
- lbExtractor
- );
-
- System.out.println("\n>>> Trained model: " + bestMdl);
-
- double accuracy = Evaluator.evaluate(
- dataCache,
- split.getTestFilter(),
- bestMdl,
- normalizationPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ minMaxScalerPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 9 (logistic regression) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(bestLearningRate),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ), bestMaxIterations, bestBatchSize, bestLocIterations, 123L);
+
+ System.out.println(">>> Perform the training to get the model.");
+ LogisticRegressionModel bestMdl = trainer.fit(
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ normalizationPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + bestMdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ split.getTestFilter(),
+ bestMdl,
+ normalizationPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 9 (logistic regression) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
[09/21] ignite git commit: IGNITE-9565 Web Console: Fixed charts
update logic. Fixed minor CSS issues.
Posted by sb...@apache.org.
IGNITE-9565 Web Console: Fixed charts update logic. Fixed minor CSS issues.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/6f39115e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/6f39115e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/6f39115e
Branch: refs/heads/ignite-gg-14206
Commit: 6f39115e3b4295c3cd8d1c517d3cd2204dd12e2e
Parents: 1ad9b9c
Author: Alexander Kalinin <ve...@yandex.ru>
Authored: Fri Sep 28 18:04:39 2018 +0700
Committer: Alexey Kuznetsov <ak...@apache.org>
Committed: Fri Sep 28 18:04:39 2018 +0700
----------------------------------------------------------------------
.../ignite-chart-series-selector/template.pug | 2 +-
.../app/components/ignite-chart/controller.js | 4 ++--
.../frontend/app/components/ignite-chart/style.scss | 16 ++++++++++++++++
3 files changed, 19 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f39115e/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug b/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
index fec0d9a..203f12f 100644
--- a/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
+++ b/modules/web-console/frontend/app/components/ignite-chart-series-selector/template.pug
@@ -24,6 +24,6 @@ button.btn-ignite.btn-ignite--link-dashed-secondary(
bs-on-before-show='$ctrl.onShow'
data-multiple='true'
ng-transclude
- ng-show='$ctrl.charts.length'
+ ng-disabled='!($ctrl.charts.length)'
)
svg(ignite-icon='gear').icon
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f39115e/modules/web-console/frontend/app/components/ignite-chart/controller.js
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart/controller.js b/modules/web-console/frontend/app/components/ignite-chart/controller.js
index 5ae536e..79156fc 100644
--- a/modules/web-console/frontend/app/components/ignite-chart/controller.js
+++ b/modules/web-console/frontend/app/components/ignite-chart/controller.js
@@ -93,8 +93,8 @@ export class IgniteChartController {
if (this.chart && _.get(changes, 'refreshRate.currentValue'))
this.onRefreshRateChanged(_.get(changes, 'refreshRate.currentValue'));
- // TODO: Investigate other signaling for resetting component state.
- if (changes.chartDataPoint && _.isNil(changes.chartDataPoint.currentValue)) {
+ if ((changes.chartDataPoint && _.isNil(changes.chartDataPoint.currentValue)) ||
+ (changes.chartHistory && _.isEmpty(changes.chartHistory.currentValue))) {
this.clearDatasets();
return;
http://git-wip-us.apache.org/repos/asf/ignite/blob/6f39115e/modules/web-console/frontend/app/components/ignite-chart/style.scss
----------------------------------------------------------------------
diff --git a/modules/web-console/frontend/app/components/ignite-chart/style.scss b/modules/web-console/frontend/app/components/ignite-chart/style.scss
index be0fb6d..3a07bd5 100644
--- a/modules/web-console/frontend/app/components/ignite-chart/style.scss
+++ b/modules/web-console/frontend/app/components/ignite-chart/style.scss
@@ -41,9 +41,25 @@ ignite-chart {
line-height: 36px;
}
+ ignite-chart-series-selector {
+ margin: 0 2px;
+ }
+
> div {
+ &:first-child {
+ width: calc(100% - 120px);
+ white-space: nowrap;
+ }
+
display: flex;
align-items: center;
+ flex-wrap: nowrap;
+ flex-grow: 0;
+
+ .chart-text {
+ overflow: hidden;
+ text-overflow: ellipsis;
+ }
}
}
[02/21] ignite git commit: IGNITE-9706: [ML] Update ignite-tensorflow
to support TensorFlow standalone client mode
Posted by sb...@apache.org.
IGNITE-9706: [ML] Update ignite-tensorflow to support
TensorFlow standalone client mode
this closes #4847
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5aef8813
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5aef8813
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5aef8813
Branch: refs/heads/ignite-gg-14206
Commit: 5aef8813269f7e7b3e3d175a4343a9fd72b68325
Parents: 66acc56
Author: Anton Dmitriev <dm...@gmail.com>
Authored: Fri Sep 28 11:49:08 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Sep 28 11:49:08 2018 +0300
----------------------------------------------------------------------
.../TensorFlowServerScriptFormatter.java | 51 ++++++++++++--------
.../util/TensorFlowUserScriptRunner.java | 15 ++----
2 files changed, 37 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/5aef8813/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
index 7cfa1c6..18854ab 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/tfrunning/TensorFlowServerScriptFormatter.java
@@ -34,10 +34,16 @@ public class TensorFlowServerScriptFormatter {
public String format(TensorFlowServer srv, boolean join, Ignite ignite) {
StringBuilder builder = new StringBuilder();
+ builder.append("from __future__ import absolute_import").append("\n");
+ builder.append("from __future__ import division").append("\n");
+ builder.append("from __future__ import print_function").append("\n");
+
builder.append("from threading import Thread").append("\n");
builder.append("from time import sleep").append("\n");
builder.append("import os, signal").append("\n");
+
builder.append("\n");
+
builder.append("def check_pid(pid):").append("\n");
builder.append(" try:").append("\n");
builder.append(" os.kill(pid, 0)").append("\n");
@@ -45,24 +51,23 @@ public class TensorFlowServerScriptFormatter {
builder.append(" return False").append("\n");
builder.append(" else:").append("\n");
builder.append(" return True").append("\n");
+
builder.append("\n");
+
builder.append("def threaded_function(pid):").append("\n");
builder.append(" while check_pid(pid):").append("\n");
builder.append(" sleep(1)").append("\n");
builder.append(" os.kill(os.getpid(), signal.SIGUSR1)").append("\n");
+
builder.append("\n");
+
builder.append("Thread(target = threaded_function, args = (int(os.environ['PPID']), )).start()")
.append("\n");
builder.append("\n");
builder.append("import tensorflow as tf").append('\n');
- builder.append("from tensorflow.contrib.ignite import IgniteDataset").append("\n");
- builder.append("\n");
- builder.append("cluster = tf.train.ClusterSpec(")
- .append(srv.getClusterSpec().format(ignite))
- .append(')')
- .append('\n');
- builder.append("");
+ builder.append("fto_import_contrib_ops = tf.contrib.resampler").append("\n");
+ builder.append("import tensorflow.contrib.igfs.python.ops.igfs_ops").append("\n");
builder.append("print('job:%s task:%d' % ('")
.append(srv.getJobName())
@@ -74,22 +79,30 @@ public class TensorFlowServerScriptFormatter {
builder.append("print('IGNITE_DATASET_PORT = ', os.environ.get('IGNITE_DATASET_PORT'))").append("\n");
builder.append("print('IGNITE_DATASET_PART = ', os.environ.get('IGNITE_DATASET_PART'))").append("\n");
- builder.append("server = tf.train.Server(cluster");
-
- if (srv.getJobName() != null)
- builder.append(", job_name=\"").append(srv.getJobName()).append('"');
-
- if (srv.getTaskIdx() != null)
- builder.append(", task_index=").append(srv.getTaskIdx());
-
- if (srv.getProto() != null)
- builder.append(", protocol=\"").append(srv.getProto()).append('"');
-
- builder.append(')').append('\n');
+ builder.append("os.environ['TF_CONFIG'] = '").append(formatTfConfigVar(srv, ignite)).append("'\n");
+ builder.append("server = tf.contrib.distribute.run_standard_tensorflow_server()").append("\n");
if (join)
builder.append("server.join()").append('\n');
return builder.toString();
}
+
+ /**
+ * Formats "TF_CONFIG" variable to be passed into user script.
+ *
+ * @param srv Server description.
+ * @param ignite Ignite instance.
+ * @return Formatted "TF_CONFIG" variable to be passed into user script.
+ */
+ private String formatTfConfigVar(TensorFlowServer srv, Ignite ignite) {
+ return "{\"cluster\" : " +
+ srv.getClusterSpec().format(ignite).replace('\n', ' ') +
+ ", " +
+ "\"task\": {\"type\" : \"" +
+ srv.getJobName() +
+ "\", \"index\": " +
+ srv.getTaskIdx() +
+ "}}";
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/5aef8813/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
----------------------------------------------------------------------
diff --git a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
index 17e63bb..d9ed9b2 100644
--- a/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
+++ b/modules/tensorflow/src/main/java/org/apache/ignite/tensorflow/cluster/util/TensorFlowUserScriptRunner.java
@@ -124,7 +124,7 @@ public class TensorFlowUserScriptRunner extends AsyncNativeProcessRunner {
Map<String, String> env = procBuilder.environment();
env.put("PYTHONPATH", workingDir.getAbsolutePath());
- env.put("TF_CONFIG", formatTfConfigVar());
+ env.put("TF_CLUSTER", formatTfClusterVar());
env.put("TF_WORKERS", formatTfWorkersVar());
env.put("TF_CHIEF_SERVER", formatTfChiefServerVar());
@@ -132,17 +132,12 @@ public class TensorFlowUserScriptRunner extends AsyncNativeProcessRunner {
}
/**
- * Formats "TF_CONFIG" variable to be passed into user script.
+ * Formats "TF_CLUSTER" variable to be passed into user script.
*
- * @return Formatted "TF_CONFIG" variable to be passed into user script.
+ * @return Formatted "TF_CLUSTER" variable to be passed into user script.
*/
- private String formatTfConfigVar() {
- return new StringBuilder()
- .append("{\"cluster\" : ")
- .append(clusterSpec.format(Ignition.ignite()))
- .append(", ")
- .append("\"task\": {\"type\" : \"" + TensorFlowClusterResolver.CHIEF_JOB_NAME + "\", \"index\": 0}}")
- .toString();
+ private String formatTfClusterVar() {
+ return clusterSpec.format(Ignition.ignite());
}
/**
[16/21] ignite git commit: IGNITE-9687: Fixed JTA tests for Java 9+.
This closes #4858.
Posted by sb...@apache.org.
IGNITE-9687: Fixed JTA tests for Java 9+. This closes #4858.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/37eedfc1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/37eedfc1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/37eedfc1
Branch: refs/heads/ignite-gg-14206
Commit: 37eedfc1824c8d2c62dbb860f08f5397c1b4033a
Parents: 4da48e6
Author: tledkov-gridgain <tl...@gridgain.com>
Authored: Fri Sep 28 17:55:14 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 28 17:55:14 2018 +0300
----------------------------------------------------------------------
modules/jta/pom.xml | 12 +++++++++++-
.../cache/GridJtaTransactionManagerSelfTest.java | 2 +-
.../jta/GridPartitionedCacheJtaFactorySelfTest.java | 2 +-
3 files changed, 13 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/37eedfc1/modules/jta/pom.xml
----------------------------------------------------------------------
diff --git a/modules/jta/pom.xml b/modules/jta/pom.xml
index 35c5582..9c3bd6c 100644
--- a/modules/jta/pom.xml
+++ b/modules/jta/pom.xml
@@ -50,7 +50,7 @@
<dependency>
<groupId>org.ow2.jotm</groupId>
<artifactId>jotm-core</artifactId>
- <version>2.1.9</version>
+ <version>2.2.3</version>
<scope>test</scope>
</dependency>
@@ -89,6 +89,16 @@
<version>6.0.43</version>
<scope>test</scope>
</dependency>
+
+ <!-- JDK9+ -->
+
+ <dependency>
+ <groupId>org.jboss.spec.javax.rmi</groupId>
+ <artifactId>jboss-rmi-api_1.0_spec</artifactId>
+ <version>1.0.6.Final</version>
+ <scope>test</scope>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/ignite/blob/37eedfc1/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
index e2ffa34..a7bb785 100644
--- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
+++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/GridJtaTransactionManagerSelfTest.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.cache;
import javax.cache.configuration.Factory;
import javax.transaction.Transaction;
+import javax.transaction.TransactionManager;
import org.apache.ignite.IgniteCache;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.TransactionConfiguration;
@@ -27,7 +28,6 @@ import org.apache.ignite.transactions.TransactionConcurrency;
import org.apache.ignite.transactions.TransactionIsolation;
import org.objectweb.jotm.Current;
import org.objectweb.jotm.Jotm;
-import org.objectweb.transaction.jta.TransactionManager;
import static org.apache.ignite.cache.CacheMode.PARTITIONED;
import static org.apache.ignite.transactions.TransactionState.ACTIVE;
http://git-wip-us.apache.org/repos/asf/ignite/blob/37eedfc1/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
index 14b7fae..ec9f8e8 100644
--- a/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
+++ b/modules/jta/src/test/java/org/apache/ignite/internal/processors/cache/jta/GridPartitionedCacheJtaFactorySelfTest.java
@@ -18,9 +18,9 @@
package org.apache.ignite.internal.processors.cache.jta;
import javax.cache.configuration.Factory;
+import javax.transaction.TransactionManager;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.configuration.TransactionConfiguration;
-import org.objectweb.transaction.jta.TransactionManager;
/**
* Factory JTA integration test using PARTITIONED cache.
[15/21] ignite git commit: IGNITE-9717: [ML] Add setters methods to
Logistic Regression and fix examples/tests
Posted by sb...@apache.org.
IGNITE-9717: [ML] Add setters methods to Logistic Regression and
fix examples/tests
this closes #4865
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4da48e6f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4da48e6f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4da48e6f
Branch: refs/heads/ignite-gg-14206
Commit: 4da48e6f90ceb7ee585b66af4f384cc868f6ca8e
Parents: a373486
Author: zaleslaw <za...@gmail.com>
Authored: Fri Sep 28 16:05:39 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Sep 28 16:05:39 2018 +0300
----------------------------------------------------------------------
.../LogisticRegressionSGDTrainerExample.java | 16 ++++---
.../ml/tutorial/Step_1_Read_and_Learn.java | 2 +-
.../examples/ml/tutorial/Step_2_Imputing.java | 2 +-
.../examples/ml/tutorial/Step_3_Categorial.java | 2 +-
.../Step_3_Categorial_with_One_Hot_Encoder.java | 2 +-
.../ml/tutorial/Step_4_Add_age_fare.java | 2 +-
.../examples/ml/tutorial/Step_5_Scaling.java | 2 +-
.../tutorial/Step_5_Scaling_with_Pipeline.java | 2 +-
.../ignite/examples/ml/tutorial/Step_6_KNN.java | 2 +-
.../ml/tutorial/Step_7_Split_train_test.java | 2 +-
.../ignite/examples/ml/tutorial/Step_8_CV.java | 2 +-
.../ml/tutorial/Step_8_CV_with_Param_Grid.java | 2 +-
.../ml/tutorial/Step_9_Go_to_LogReg.java | 27 ++++++-----
.../ml/tutorial/TutorialStepByStepExample.java | 2 +-
.../binomial/LogisticRegressionSGDTrainer.java | 47 ++++++++++----------
.../LogRegressionMultiClassTrainer.java | 29 +++++++-----
.../SVMLinearBinaryClassificationTrainer.java | 2 +-
...VMLinearMultiClassClassificationTrainer.java | 2 +-
.../apache/ignite/ml/pipeline/PipelineTest.java | 18 +++-----
.../logistic/LogRegMultiClassTrainerTest.java | 1 -
.../logistic/LogisticRegressionModelTest.java | 17 +++----
.../LogisticRegressionSGDTrainerTest.java | 24 +++++-----
22 files changed, 111 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
index 8d4218d..15330d0 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
@@ -60,11 +60,16 @@ public class LogisticRegressionSGDTrainerExample {
IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
System.out.println(">>> Create new logistic regression trainer object.");
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), 100000, 10, 100, 123L);
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ))
+ .withMaxIterations(100000)
+ .withLocIterations(100)
+ .withBatchSize(10)
+ .withSeed(123L);
System.out.println(">>> Perform the training to get the model.");
LogisticRegressionModel mdl = trainer.fit(
@@ -218,5 +223,4 @@ public class LogisticRegressionSGDTrainerExample {
{1, 5.1, 2.5, 3, 1.1},
{1, 5.7, 2.8, 4.1, 1.3},
};
-
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
index 264dbf4..481fa1d 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
@@ -42,7 +42,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_1_Read_and_Learn {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 1 (read and learn) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
index df73235..d60dc4b 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
@@ -44,7 +44,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_2_Imputing {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 2 (imputing) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
index 463a6ba..ac2fe08 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
@@ -47,7 +47,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_3_Categorial {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 3 (categorial) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
index 93e7e79..f0b6efe 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
@@ -48,7 +48,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_3_Categorial_with_One_Hot_Encoder {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 3 (categorial with One-hot encoder) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
index bbeedb6..71e9efd 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
@@ -45,7 +45,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_4_Add_age_fare {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 4 (add age and fare) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
index 7d934d7..fe7bf91 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
@@ -48,7 +48,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_5_Scaling {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 5 (scaling) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
index cc0a278..bd7cc21 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
@@ -48,7 +48,7 @@ import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
*/
public class Step_5_Scaling_with_Pipeline {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 5 (scaling) via Pipeline example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
index 0c8b562..a35b841 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_6_KNN.java
@@ -49,7 +49,7 @@ import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
*/
public class Step_6_KNN {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 6 (kNN) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
index c6d033c..53d4d0a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_7_Split_train_test.java
@@ -51,7 +51,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_7_Split_train_test {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 7 (split to train and test) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
index d83e14a..feedccf 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV.java
@@ -63,7 +63,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_8_CV {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 8 (cross-validation) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
index 594c0eb..670f025 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_8_CV_with_Param_Grid.java
@@ -65,7 +65,7 @@ import org.apache.ignite.ml.tree.DecisionTreeNode;
*/
public class Step_8_CV_with_Param_Grid {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 8 (cross-validation with param grid) example started.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
index 4e1e005..b98b0eb 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_9_Go_to_LogReg.java
@@ -56,7 +56,7 @@ import org.apache.ignite.ml.selection.split.TrainTestSplit;
*/
public class Step_9_Go_to_LogReg {
/** Run example. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
System.out.println();
System.out.println(">>> Tutorial step 9 (logistic regression) example started.");
@@ -124,12 +124,13 @@ public class Step_9_Go_to_LogReg {
minMaxScalerPreprocessor
);
- LogisticRegressionSGDTrainer<?> trainer
- = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(learningRate),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), maxIterations, batchSize, locIterations, 123L);
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(new SimpleGDUpdateCalculator(learningRate),
+ SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg))
+ .withMaxIterations(maxIterations)
+ .withLocIterations(locIterations)
+ .withBatchSize(batchSize)
+ .withSeed(123L);
CrossValidation<LogisticRegressionModel, Double, Integer, Object[]>
scoreCalculator = new CrossValidation<>();
@@ -187,11 +188,13 @@ public class Step_9_Go_to_LogReg {
minMaxScalerPreprocessor
);
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(bestLearningRate),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), bestMaxIterations, bestBatchSize, bestLocIterations, 123L);
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(new SimpleGDUpdateCalculator(bestLearningRate),
+ SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg))
+ .withMaxIterations(bestMaxIterations)
+ .withLocIterations(bestLocIterations)
+ .withBatchSize(bestBatchSize)
+ .withSeed(123L);
System.out.println(">>> Perform the training to get the model.");
LogisticRegressionModel bestMdl = trainer.fit(
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
index 67f4bf5..a376ae6 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/TutorialStepByStepExample.java
@@ -23,7 +23,7 @@ package org.apache.ignite.examples.ml.tutorial;
*/
public class TutorialStepByStepExample {
/** Run examples with default settings. */
- public static void main(String[] args) throws InterruptedException {
+ public static void main(String[] args) {
Step_1_Read_and_Learn.main(args);
Step_2_Imputing.main(args);
Step_3_Categorial.main(args);
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
index fb5d5a0..74a296d 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/binomial/LogisticRegressionSGDTrainer.java
@@ -33,6 +33,8 @@ import org.apache.ignite.ml.nn.MultilayerPerceptron;
import org.apache.ignite.ml.nn.UpdatesStrategy;
import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
import org.jetbrains.annotations.NotNull;
@@ -41,37 +43,23 @@ import org.jetbrains.annotations.NotNull;
*/
public class LogisticRegressionSGDTrainer<P extends Serializable> extends SingleLabelDatasetTrainer<LogisticRegressionModel> {
/** Update strategy. */
- private UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
+ private UpdatesStrategy updatesStgy = new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ );
/** Max number of iteration. */
- private int maxIterations;
+ private int maxIterations = 100;
/** Batch size. */
- private int batchSize;
+ private int batchSize = 100;
/** Number of local iterations. */
- private int locIterations;
+ private int locIterations = 100;
/** Seed for random generator. */
- private long seed;
-
- /**
- * Constructs a new instance of linear regression SGD trainer.
- *
- * @param updatesStgy Update strategy.
- * @param maxIterations Max number of iteration.
- * @param batchSize Batch size.
- * @param locIterations Number of local iterations.
- * @param seed Seed for random generator.
- */
- public LogisticRegressionSGDTrainer(UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy, int maxIterations,
- int batchSize, int locIterations, long seed) {
- this.updatesStgy = updatesStgy;
- this.maxIterations = maxIterations;
- this.batchSize = batchSize;
- this.locIterations = locIterations;
- this.seed = seed;
- }
+ private long seed = 1234L;
/** {@inheritDoc} */
@Override public <K, V> LogisticRegressionModel fit(DatasetBuilder<K, V> datasetBuilder,
@@ -202,11 +190,22 @@ public class LogisticRegressionSGDTrainer<P extends Serializable> extends Single
}
/**
+ * Set up the regularization parameter.
+ *
+ * @param updatesStgy Update strategy.
+ * @return Trainer with new update strategy parameter value.
+ */
+ public LogisticRegressionSGDTrainer withUpdatesStgy(UpdatesStrategy updatesStgy) {
+ this.updatesStgy = updatesStgy;
+ return this;
+ }
+
+ /**
* Get the update strategy.
*
* @return The property value.
*/
- public UpdatesStrategy<? super MultilayerPerceptron, P> getUpdatesStgy() {
+ public UpdatesStrategy getUpdatesStgy() {
return updatesStgy;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
index b9cdcc7..71d54fa 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/regressions/logistic/multiclass/LogRegressionMultiClassTrainer.java
@@ -32,8 +32,9 @@ import org.apache.ignite.ml.dataset.PartitionDataBuilder;
import org.apache.ignite.ml.dataset.primitive.context.EmptyContext;
import org.apache.ignite.ml.math.functions.IgniteBiFunction;
import org.apache.ignite.ml.math.primitives.vector.Vector;
-import org.apache.ignite.ml.nn.MultilayerPerceptron;
import org.apache.ignite.ml.nn.UpdatesStrategy;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
+import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
import org.apache.ignite.ml.regressions.logistic.binomial.LogisticRegressionModel;
import org.apache.ignite.ml.regressions.logistic.binomial.LogisticRegressionSGDTrainer;
import org.apache.ignite.ml.structures.partition.LabelPartitionDataBuilderOnHeap;
@@ -46,19 +47,23 @@ import org.apache.ignite.ml.trainers.SingleLabelDatasetTrainer;
public class LogRegressionMultiClassTrainer<P extends Serializable>
extends SingleLabelDatasetTrainer<LogRegressionMultiClassModel> {
/** Update strategy. */
- private UpdatesStrategy<? super MultilayerPerceptron, P> updatesStgy;
+ private UpdatesStrategy updatesStgy = new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ );
/** Max number of iteration. */
- private int amountOfIterations;
+ private int amountOfIterations = 100;
/** Batch size. */
- private int batchSize;
+ private int batchSize = 100;
/** Number of local iterations. */
- private int amountOfLocIterations;
+ private int amountOfLocIterations = 100;
/** Seed for random generator. */
- private long seed;
+ private long seed = 1234L;
/**
* Trains model based on the specified data.
@@ -90,7 +95,11 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
classes.forEach(clsLb -> {
LogisticRegressionSGDTrainer<?> trainer =
- new LogisticRegressionSGDTrainer<>(updatesStgy, amountOfIterations, batchSize, amountOfLocIterations, seed);
+ new LogisticRegressionSGDTrainer<>()
+ .withBatchSize(batchSize)
+ .withLocIterations(amountOfLocIterations)
+ .withMaxIterations(amountOfIterations)
+ .withSeed(seed);
IgniteBiFunction<K, V, Double> lbTransformer = (k, v) -> {
Double lb = lbExtractor.apply(k, v);
@@ -238,7 +247,7 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Set up the regularization parameter.
+ * Set up the updates strategy.
*
* @param updatesStgy Update strategy.
* @return Trainer with new update strategy parameter value.
@@ -249,11 +258,11 @@ public class LogRegressionMultiClassTrainer<P extends Serializable>
}
/**
- * Get the update strategy..
+ * Get the update strategy.
*
* @return The parameter value.
*/
- public UpdatesStrategy<? super MultilayerPerceptron, P> getUpdatesStgy() {
+ public UpdatesStrategy getUpdatesStgy() {
return updatesStgy;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
index 2c621c8..47666f4 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearBinaryClassificationTrainer.java
@@ -50,7 +50,7 @@ public class SVMLinearBinaryClassificationTrainer extends SingleLabelDatasetTrai
private double lambda = 0.4;
/** The seed number. */
- private long seed;
+ private long seed = 1234L;
/**
* Trains model based on the specified data.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
index ec60034..b161914 100644
--- a/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
+++ b/modules/ml/src/main/java/org/apache/ignite/ml/svm/SVMLinearMultiClassClassificationTrainer.java
@@ -52,7 +52,7 @@ public class SVMLinearMultiClassClassificationTrainer
private double lambda = 0.2;
/** The seed number. */
- private long seed;
+ private long seed = 1234L;
/**
* Trains model based on the specified data.
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/test/java/org/apache/ignite/ml/pipeline/PipelineTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/pipeline/PipelineTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/pipeline/PipelineTest.java
index 91bbcd4..d517ce6 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/pipeline/PipelineTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/pipeline/PipelineTest.java
@@ -51,11 +51,13 @@ public class PipelineTest extends TrainerTest {
cacheMock.put(i, convertedRow);
}
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator().withLearningRate(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), 100000, 10, 100, 123L);
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg))
+ .withMaxIterations(100000)
+ .withLocIterations(100)
+ .withBatchSize(10)
+ .withSeed(123L);
PipelineMdl<Integer, Double[]> mdl = new Pipeline<Integer, Double[], Vector>()
.addFeatureExtractor((k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)))
@@ -88,12 +90,6 @@ public class PipelineTest extends TrainerTest {
cacheMock.put(i, convertedRow);
}
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator().withLearningRate(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), 100000, 10, 100, 123L);
-
PipelineMdl<Integer, Double[]> mdl = new Pipeline<Integer, Double[], Vector>()
.addFeatureExtractor((k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)))
.addLabelExtractor((k, v) -> v[0])
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
index 78cd08d..c99bf02 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogRegMultiClassTrainerTest.java
@@ -133,7 +133,6 @@ public class LogRegMultiClassTrainerTest extends TrainerTest {
VectorUtils.of(10, -10)
);
-
for (Vector vec : vectors) {
TestUtils.assertEquals(originalMdl.apply(vec), updatedOnSameDS.apply(vec), PRECISION);
TestUtils.assertEquals(originalMdl.apply(vec), updatedOnEmptyDS.apply(vec), PRECISION);
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModelTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModelTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModelTest.java
index 89c9cca..e8aaacd 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModelTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionModelTest.java
@@ -38,7 +38,7 @@ public class LogisticRegressionModelTest {
/** */
@Test
public void testPredict() {
- Vector weights = new DenseVector(new double[]{2.0, 3.0});
+ Vector weights = new DenseVector(new double[] {2.0, 3.0});
assertFalse(new LogisticRegressionModel(weights, 1.0).isKeepingRawLabels());
@@ -57,35 +57,36 @@ public class LogisticRegressionModelTest {
/** */
@Test(expected = CardinalityException.class)
public void testPredictOnAnObservationWithWrongCardinality() {
- Vector weights = new DenseVector(new double[]{2.0, 3.0});
+ Vector weights = new DenseVector(new double[] {2.0, 3.0});
LogisticRegressionModel mdl = new LogisticRegressionModel(weights, 1.0);
- Vector observation = new DenseVector(new double[]{1.0});
+ Vector observation = new DenseVector(new double[] {1.0});
mdl.apply(observation);
}
/** */
private void verifyPredict(LogisticRegressionModel mdl) {
- Vector observation = new DenseVector(new double[]{1.0, 1.0});
+ Vector observation = new DenseVector(new double[] {1.0, 1.0});
TestUtils.assertEquals(sigmoid(1.0 + 2.0 * 1.0 + 3.0 * 1.0), mdl.apply(observation), PRECISION);
- observation = new DenseVector(new double[]{2.0, 1.0});
+ observation = new DenseVector(new double[] {2.0, 1.0});
TestUtils.assertEquals(sigmoid(1.0 + 2.0 * 2.0 + 3.0 * 1.0), mdl.apply(observation), PRECISION);
- observation = new DenseVector(new double[]{1.0, 2.0});
+ observation = new DenseVector(new double[] {1.0, 2.0});
TestUtils.assertEquals(sigmoid(1.0 + 2.0 * 1.0 + 3.0 * 2.0), mdl.apply(observation), PRECISION);
- observation = new DenseVector(new double[]{-2.0, 1.0});
+ observation = new DenseVector(new double[] {-2.0, 1.0});
TestUtils.assertEquals(sigmoid(1.0 - 2.0 * 2.0 + 3.0 * 1.0), mdl.apply(observation), PRECISION);
- observation = new DenseVector(new double[]{1.0, -2.0});
+ observation = new DenseVector(new double[] {1.0, -2.0});
TestUtils.assertEquals(sigmoid(1.0 + 2.0 * 1.0 - 3.0 * 2.0), mdl.apply(observation), PRECISION);
}
/**
* Sigmoid function.
+ *
* @param z The regression value.
* @return The result.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/4da48e6f/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
----------------------------------------------------------------------
diff --git a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
index 723677c..d9b6f7a 100644
--- a/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
+++ b/modules/ml/src/test/java/org/apache/ignite/ml/regressions/logistic/LogisticRegressionSGDTrainerTest.java
@@ -45,11 +45,13 @@ public class LogisticRegressionSGDTrainerTest extends TrainerTest {
for (int i = 0; i < twoLinearlySeparableClasses.length; i++)
cacheMock.put(i, twoLinearlySeparableClasses[i]);
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator().withLearningRate(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), 100000, 10, 100, 123L);
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg))
+ .withMaxIterations(100000)
+ .withLocIterations(100)
+ .withBatchSize(10)
+ .withSeed(123L);
LogisticRegressionModel mdl = trainer.fit(
cacheMock,
@@ -70,11 +72,13 @@ public class LogisticRegressionSGDTrainerTest extends TrainerTest {
for (int i = 0; i < twoLinearlySeparableClasses.length; i++)
cacheMock.put(i, twoLinearlySeparableClasses[i]);
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator().withLearningRate(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), 100000, 10, 100, 123L);
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal, SimpleGDParameterUpdate::avg))
+ .withMaxIterations(100000)
+ .withLocIterations(100)
+ .withBatchSize(10)
+ .withSeed(123L);
LogisticRegressionModel originalMdl = trainer.fit(
cacheMock,
[19/21] ignite git commit: IGNITE-9731 Fixed NPE on concurrent WAL
flush - Fixes #4863.
Posted by sb...@apache.org.
IGNITE-9731 Fixed NPE on concurrent WAL flush - Fixes #4863.
Signed-off-by: Alexey Goncharuk <al...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/69adfd5e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/69adfd5e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/69adfd5e
Branch: refs/heads/ignite-gg-14206
Commit: 69adfd5e94530457525bd0403182f577b418ca95
Parents: 036bd07
Author: Anton Kalashnikov <ka...@yandex.ru>
Authored: Fri Sep 28 18:20:00 2018 +0300
Committer: Alexey Goncharuk <al...@gmail.com>
Committed: Fri Sep 28 18:22:48 2018 +0300
----------------------------------------------------------------------
.../wal/FileWriteAheadLogManager.java | 10 +-
.../db/wal/WalRolloverRecordLoggingTest.java | 156 +++++++++++++++++++
.../IgnitePdsWithIndexingCoreTestSuite.java | 2 +
3 files changed, 165 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/69adfd5e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
index 43dfb8f..8388bb2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/FileWriteAheadLogManager.java
@@ -2412,19 +2412,23 @@ public class FileWriteAheadLogManager extends GridCacheSharedManagerAdapter impl
private abstract static class FileHandle {
/** I/O interface for read/write operations with file */
SegmentIO fileIO;
+
+ /** Segment idx corresponded to fileIo*/
+ final long segmentIdx;
/**
- * @param fileIO I/O interface for read/write operations of FileHandle. *
+ * @param fileIO I/O interface for read/write operations of FileHandle.
*/
- private FileHandle(SegmentIO fileIO) {
+ private FileHandle(@NotNull SegmentIO fileIO) {
this.fileIO = fileIO;
+ segmentIdx = fileIO.getSegmentId();
}
/**
* @return Absolute WAL segment file index (incremental counter).
*/
public long getSegmentId(){
- return fileIO.getSegmentId();
+ return segmentIdx;
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/69adfd5e/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
new file mode 100644
index 0000000..67caf63
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/WalRolloverRecordLoggingTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.persistence.db.wal;
+
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.IgniteCache;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.configuration.DataRegionConfiguration;
+import org.apache.ignite.configuration.DataStorageConfiguration;
+import org.apache.ignite.configuration.IgniteConfiguration;
+import org.apache.ignite.failure.StopNodeOrHaltFailureHandler;
+import org.apache.ignite.internal.IgniteEx;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.IgniteInterruptedCheckedException;
+import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager;
+import org.apache.ignite.internal.pagemem.wal.record.CheckpointRecord;
+import org.apache.ignite.internal.processors.cache.persistence.IgniteCacheDatabaseSharedManager;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.TcpDiscoveryIpFinder;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder;
+import org.apache.ignite.testframework.GridTestUtils;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+
+import static org.apache.ignite.configuration.DataStorageConfiguration.DFLT_WAL_PATH;
+import static org.apache.ignite.configuration.WALMode.LOG_ONLY;
+/**
+ *
+ */
+public class WalRolloverRecordLoggingTest extends GridCommonAbstractTest {
+ /** */
+ private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
+
+ /** */
+ private static class RolloverRecord extends CheckpointRecord {
+ /** */
+ private RolloverRecord() {
+ super(null);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean rollOver() {
+ return true;
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override protected IgniteConfiguration getConfiguration(String name) throws Exception {
+ IgniteConfiguration cfg = super.getConfiguration(name);
+
+ cfg.setDiscoverySpi(new TcpDiscoverySpi().setIpFinder(IP_FINDER));
+
+ cfg.setDataStorageConfiguration(new DataStorageConfiguration()
+ .setDefaultDataRegionConfiguration(new DataRegionConfiguration()
+ .setPersistenceEnabled(true)
+ .setMaxSize(40 * 1024 * 1024))
+ .setWalMode(LOG_ONLY)
+ .setWalSegmentSize(4 * 1024 * 1024)
+ .setWalArchivePath(DFLT_WAL_PATH));
+
+ cfg.setFailureHandler(new StopNodeOrHaltFailureHandler(false, 0));
+
+ return cfg;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void beforeTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ stopAllGrids();
+
+ cleanPersistenceDir();
+ }
+
+ /** */
+ public void testAvoidInfinityWaitingOnRolloverOfSegment() throws Exception {
+ IgniteEx ig = startGrid(0);
+
+ ig.cluster().active(true);
+
+ IgniteCache<Integer, Integer> cache = ig.getOrCreateCache(DEFAULT_CACHE_NAME);
+
+ long startTime = U.currentTimeMillis();
+ long duration = 5_000;
+
+ IgniteInternalFuture<Long> fut = GridTestUtils.runMultiThreadedAsync(
+ () -> {
+ ThreadLocalRandom random = ThreadLocalRandom.current();
+
+ while (U.currentTimeMillis() - startTime < duration)
+ cache.put(random.nextInt(100_000), random.nextInt(100_000));
+ },
+ 8, "cache-put-thread");
+
+ Thread t = new Thread(() -> {
+ do {
+ try {
+ U.sleep(100);
+ }
+ catch (IgniteInterruptedCheckedException e) {
+ // No-op.
+ }
+
+ ig.context().cache().context().database().wakeupForCheckpoint("test");
+ } while (U.currentTimeMillis() - startTime < duration);
+ });
+
+ t.start();
+
+ IgniteWriteAheadLogManager walMgr = ig.context().cache().context().wal();
+
+ IgniteCacheDatabaseSharedManager dbMgr = ig.context().cache().context().database();
+
+ RolloverRecord rec = new RolloverRecord();
+
+ do {
+ try {
+ dbMgr.checkpointReadLock();
+
+ try {
+ walMgr.log(rec);
+ }
+ finally {
+ dbMgr.checkpointReadUnlock();
+ }
+ }
+ catch (IgniteCheckedException e) {
+ log.error(e.getMessage(), e);
+ }
+ } while (U.currentTimeMillis() - startTime < duration);
+
+ fut.get();
+
+ t.join();
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/69adfd5e/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
index 2989ccd..caea388 100644
--- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
+++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingCoreTestSuite.java
@@ -40,6 +40,7 @@ import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalR
import org.apache.ignite.internal.processors.cache.persistence.db.wal.IgniteWalRecoveryWithCompactionTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalPathsTest;
import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRecoveryTxLogicalRecordsTest;
+import org.apache.ignite.internal.processors.cache.persistence.db.wal.WalRolloverRecordLoggingTest;
/**
* Test suite for tests that cover core PDS features and depend on indexing module.
@@ -59,6 +60,7 @@ public class IgnitePdsWithIndexingCoreTestSuite extends TestSuite {
suite.addTestSuite(PersistenceDirectoryWarningLoggingTest.class);
suite.addTestSuite(WalPathsTest.class);
suite.addTestSuite(WalRecoveryTxLogicalRecordsTest.class);
+ suite.addTestSuite(WalRolloverRecordLoggingTest.class);
suite.addTestSuite(IgniteWalRecoveryTest.class);
suite.addTestSuite(IgniteWalRecoveryWithCompactionTest.class);
[08/21] ignite git commit: IGNITE-7282 Add missing license header
Posted by sb...@apache.org.
IGNITE-7282 Add missing license header
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/1ad9b9c4
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/1ad9b9c4
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/1ad9b9c4
Branch: refs/heads/ignite-gg-14206
Commit: 1ad9b9c4a42c04f233744094d500782915d1cba9
Parents: d5f6e50
Author: Pavel Tupitsyn <pt...@apache.org>
Authored: Fri Sep 28 13:47:35 2018 +0300
Committer: Pavel Tupitsyn <pt...@apache.org>
Committed: Fri Sep 28 13:47:35 2018 +0300
----------------------------------------------------------------------
.../Client/EndpointTest.cs | 17 +++++++++++++++++
1 file changed, 17 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/1ad9b9c4/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/EndpointTest.cs
----------------------------------------------------------------------
diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/EndpointTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/EndpointTest.cs
index 05366ed..7b2e755 100644
--- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/EndpointTest.cs
+++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/EndpointTest.cs
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
namespace Apache.Ignite.Core.Tests.Client
{
using System.Linq;
[05/21] ignite git commit: IGNITE-9711: [ML] Remove IgniteThread
wrapper from ml examples
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
index b7ca448..520b8cc 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/multiclass/SVMMultiClassClassificationExample.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearMultiClassClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run SVM multi-class classification trainer ({@link SVMLinearMultiClassClassificationModel}) over distributed dataset
@@ -59,106 +58,100 @@ public class SVMMultiClassClassificationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- SVMMultiClassClassificationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, Vector> dataCache = new TestCache(ignite).getVectors(data);
+ IgniteCache<Integer, Vector> dataCache = new TestCache(ignite).getVectors(data);
- SVMLinearMultiClassClassificationTrainer trainer = new SVMLinearMultiClassClassificationTrainer();
+ SVMLinearMultiClassClassificationTrainer trainer = new SVMLinearMultiClassClassificationTrainer();
- SVMLinearMultiClassClassificationModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> {
- double[] arr = v.asArray();
- return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
- },
- (k, v) -> v.get(0)
- );
+ SVMLinearMultiClassClassificationModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> {
+ double[] arr = v.asArray();
+ return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
+ },
+ (k, v) -> v.get(0)
+ );
- System.out.println(">>> SVM Multi-class model");
- System.out.println(mdl.toString());
+ System.out.println(">>> SVM Multi-class model");
+ System.out.println(mdl.toString());
- MinMaxScalerTrainer<Integer, Vector> normalizationTrainer = new MinMaxScalerTrainer<>();
+ MinMaxScalerTrainer<Integer, Vector> normalizationTrainer = new MinMaxScalerTrainer<>();
- IgniteBiFunction<Integer, Vector, Vector> preprocessor = normalizationTrainer.fit(
- ignite,
- dataCache,
- (k, v) -> {
- double[] arr = v.asArray();
- return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
- }
- );
-
- SVMLinearMultiClassClassificationModel mdlWithNormalization = trainer.fit(
- ignite,
- dataCache,
- preprocessor,
- (k, v) -> v.get(0)
- );
+ IgniteBiFunction<Integer, Vector, Vector> preprocessor = normalizationTrainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> {
+ double[] arr = v.asArray();
+ return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
+ }
+ );
- System.out.println(">>> SVM Multi-class model with minmaxscaling");
- System.out.println(mdlWithNormalization.toString());
+ SVMLinearMultiClassClassificationModel mdlWithNormalization = trainer.fit(
+ ignite,
+ dataCache,
+ preprocessor,
+ (k, v) -> v.get(0)
+ );
- System.out.println(">>> ----------------------------------------------------------------");
- System.out.println(">>> | Prediction\t| Prediction with Normalization\t| Ground Truth\t|");
- System.out.println(">>> ----------------------------------------------------------------");
+ System.out.println(">>> SVM Multi-class model with minmaxscaling");
+ System.out.println(mdlWithNormalization.toString());
- int amountOfErrors = 0;
- int amountOfErrorsWithNormalization = 0;
- int totalAmount = 0;
+ System.out.println(">>> ----------------------------------------------------------------");
+ System.out.println(">>> | Prediction\t| Prediction with Normalization\t| Ground Truth\t|");
+ System.out.println(">>> ----------------------------------------------------------------");
- // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
- int[][] confusionMtx = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
- int[][] confusionMtxWithNormalization = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
+ int amountOfErrors = 0;
+ int amountOfErrorsWithNormalization = 0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, Vector>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, Vector> observation : observations) {
- double[] val = observation.getValue().asArray();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+ int[][] confusionMtx = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
+ int[][] confusionMtxWithNormalization = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
- double prediction = mdl.apply(new DenseVector(inputs));
- double predictionWithNormalization = mdlWithNormalization.apply(new DenseVector(inputs));
+ try (QueryCursor<Cache.Entry<Integer, Vector>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, Vector> observation : observations) {
+ double[] val = observation.getValue().asArray();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- totalAmount++;
+ double prediction = mdl.apply(new DenseVector(inputs));
+ double predictionWithNormalization = mdlWithNormalization.apply(new DenseVector(inputs));
- // Collect data for model
- if(groundTruth != prediction)
- amountOfErrors++;
+ totalAmount++;
- int idx1 = (int)prediction == 1 ? 0 : ((int)prediction == 3 ? 1 : 2);
- int idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
+ // Collect data for model
+ if(groundTruth != prediction)
+ amountOfErrors++;
- confusionMtx[idx1][idx2]++;
+ int idx1 = (int)prediction == 1 ? 0 : ((int)prediction == 3 ? 1 : 2);
+ int idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
- // Collect data for model with minmaxscaling
- if(groundTruth != predictionWithNormalization)
- amountOfErrorsWithNormalization++;
+ confusionMtx[idx1][idx2]++;
- idx1 = (int)predictionWithNormalization == 1 ? 0 : ((int)predictionWithNormalization == 3 ? 1 : 2);
- idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
+ // Collect data for model with minmaxscaling
+ if(groundTruth != predictionWithNormalization)
+ amountOfErrorsWithNormalization++;
- confusionMtxWithNormalization[idx1][idx2]++;
+ idx1 = (int)predictionWithNormalization == 1 ? 0 : ((int)predictionWithNormalization == 3 ? 1 : 2);
+ idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t\t\t\t\t| %.4f\t\t|\n", prediction, predictionWithNormalization, groundTruth);
- }
- System.out.println(">>> ----------------------------------------------------------------");
- System.out.println("\n>>> -----------------SVM model-------------");
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
- System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+ confusionMtxWithNormalization[idx1][idx2]++;
- System.out.println("\n>>> -----------------SVM model with Normalization-------------");
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrorsWithNormalization);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrorsWithNormalization / (double)totalAmount));
- System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtxWithNormalization));
-
- System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t\t\t\t\t| %.4f\t\t|\n", prediction, predictionWithNormalization, groundTruth);
}
- });
+ System.out.println(">>> ----------------------------------------------------------------");
+ System.out.println("\n>>> -----------------SVM model-------------");
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+
+ System.out.println("\n>>> -----------------SVM model with Normalization-------------");
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrorsWithNormalization);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrorsWithNormalization / (double)totalAmount));
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtxWithNormalization));
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
index 28a5fbc..652b293 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeClassificationTrainerExample.java
@@ -26,7 +26,6 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Example of using distributed {@link DecisionTreeClassificationTrainer}.
@@ -53,58 +52,49 @@ public class DecisionTreeClassificationTrainerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- DecisionTreeClassificationTrainerExample.class.getSimpleName(), () -> {
+ // Create cache with training data.
+ CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
- // Create cache with training data.
- CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
- trainingSetCfg.setName("TRAINING_SET");
- trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
- IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
+ Random rnd = new Random(0);
- Random rnd = new Random(0);
+ // Fill training data.
+ for (int i = 0; i < 1000; i++)
+ trainingSet.put(i, generatePoint(rnd));
- // Fill training data.
- for (int i = 0; i < 1000; i++)
- trainingSet.put(i, generatePoint(rnd));
+ // Create classification trainer.
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
- // Create classification trainer.
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ trainingSet,
+ (k, v) -> VectorUtils.of(v.x, v.y),
+ (k, v) -> v.lb
+ );
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
- ignite,
- trainingSet,
- (k, v) -> VectorUtils.of(v.x, v.y),
- (k, v) -> v.lb
- );
+ System.out.println(">>> Decision tree classification model: " + mdl);
- System.out.println(">>> Decision tree classification model: " + mdl);
+ // Calculate score.
+ int correctPredictions = 0;
+ for (int i = 0; i < 1000; i++) {
+ LabeledPoint pnt = generatePoint(rnd);
- // Calculate score.
- int correctPredictions = 0;
- for (int i = 0; i < 1000; i++) {
- LabeledPoint pnt = generatePoint(rnd);
+ double prediction = mdl.apply(VectorUtils.of(pnt.x, pnt.y));
+ double lbl = pnt.lb;
- double prediction = mdl.apply(VectorUtils.of(pnt.x, pnt.y));
- double lbl = pnt.lb;
+ if (i %50 == 1)
+ System.out.printf(">>> test #: %d\t\t predicted: %.4f\t\tlabel: %.4f\n", i, prediction, lbl);
- if (i %50 == 1)
- System.out.printf(">>> test #: %d\t\t predicted: %.4f\t\tlabel: %.4f\n", i, prediction, lbl);
+ if (prediction == lbl)
+ correctPredictions++;
+ }
- if (prediction == lbl)
- correctPredictions++;
- }
-
- System.out.println(">>> Accuracy: " + correctPredictions / 10.0 + "%");
-
- System.out.println(">>> Decision tree classification trainer example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> Accuracy: " + correctPredictions / 10.0 + "%");
+ System.out.println(">>> Decision tree classification trainer example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
index 301df10..2a89c7e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/DecisionTreeRegressionTrainerExample.java
@@ -25,7 +25,6 @@ import org.apache.ignite.configuration.CacheConfiguration;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.tree.DecisionTreeNode;
import org.apache.ignite.ml.tree.DecisionTreeRegressionTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* Example of using distributed {@link DecisionTreeRegressionTrainer}.
@@ -53,51 +52,43 @@ public class DecisionTreeRegressionTrainerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- DecisionTreeRegressionTrainerExample.class.getSimpleName(), () -> {
+ // Create cache with training data.
+ CacheConfiguration<Integer, Point> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
- // Create cache with training data.
- CacheConfiguration<Integer, Point> trainingSetCfg = new CacheConfiguration<>();
- trainingSetCfg.setName("TRAINING_SET");
- trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ IgniteCache<Integer, Point> trainingSet = ignite.createCache(trainingSetCfg);
- IgniteCache<Integer, Point> trainingSet = ignite.createCache(trainingSetCfg);
+ // Fill training data.
+ generatePoints(trainingSet);
- // Fill training data.
- generatePoints(trainingSet);
+ // Create regression trainer.
+ DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(10, 0);
- // Create regression trainer.
- DecisionTreeRegressionTrainer trainer = new DecisionTreeRegressionTrainer(10, 0);
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ trainingSet,
+ (k, v) -> VectorUtils.of(v.x),
+ (k, v) -> v.y
+ );
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
- ignite,
- trainingSet,
- (k, v) -> VectorUtils.of(v.x),
- (k, v) -> v.y
- );
+ System.out.println(">>> Decision tree regression model: " + mdl);
- System.out.println(">>> Decision tree regression model: " + mdl);
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
+ // Calculate score.
+ for (int x = 0; x < 10; x++) {
+ double predicted = mdl.apply(VectorUtils.of(x));
- // Calculate score.
- for (int x = 0; x < 10; x++) {
- double predicted = mdl.apply(VectorUtils.of(x));
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x));
+ }
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x));
- }
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> ---------------------------------");
-
- System.out.println(">>> Decision tree regression trainer example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> Decision tree regression trainer example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
index e092e5c..5beb954 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesClassificationTrainerExample.java
@@ -27,7 +27,6 @@ import org.apache.ignite.ml.composition.boosting.convergence.mean.MeanAbsValueCo
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.trainers.DatasetTrainer;
import org.apache.ignite.ml.tree.boosting.GDBBinaryClassifierOnTreesTrainer;
-import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.NotNull;
/**
@@ -50,45 +49,38 @@ public class GDBOnTreesClassificationTrainerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- GDBBinaryClassifierOnTreesTrainer.class.getSimpleName(), () -> {
+ // Create cache with training data.
+ CacheConfiguration<Integer, double[]> trainingSetCfg = createCacheConfiguration();
+ IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
- // Create cache with training data.
- CacheConfiguration<Integer, double[]> trainingSetCfg = createCacheConfiguration();
- IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
+ // Create regression trainer.
+ DatasetTrainer<ModelsComposition, Double> trainer = new GDBBinaryClassifierOnTreesTrainer(1.0, 300, 2, 0.)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.1));
- // Create regression trainer.
- DatasetTrainer<ModelsComposition, Double> trainer = new GDBBinaryClassifierOnTreesTrainer(1.0, 300, 2, 0.)
- .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.1));
+ // Train decision tree model.
+ ModelsComposition mdl = trainer.fit(
+ ignite,
+ trainingSet,
+ (k, v) -> VectorUtils.of(v[0]),
+ (k, v) -> v[1]
+ );
- // Train decision tree model.
- ModelsComposition mdl = trainer.fit(
- ignite,
- trainingSet,
- (k, v) -> VectorUtils.of(v[0]),
- (k, v) -> v[1]
- );
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Valid answer\t|");
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Valid answer\t|");
- System.out.println(">>> ---------------------------------");
+ // Calculate score.
+ for (int x = -5; x < 5; x++) {
+ double predicted = mdl.apply(VectorUtils.of(x));
- // Calculate score.
- for (int x = -5; x < 5; x++) {
- double predicted = mdl.apply(VectorUtils.of(x));
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x) < 0 ? 0.0 : 1.0);
+ }
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.sin(x) < 0 ? 0.0 : 1.0);
- }
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> Count of trees = " + mdl.getModels().size());
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> Count of trees = " + mdl.getModels().size());
- System.out.println(">>> ---------------------------------");
-
- System.out.println(">>> GDB classification trainer example completed.");
- });
-
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> GDB classification trainer example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
index 3662973..482a47c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/boosting/GDBOnTreesRegressionTrainerExample.java
@@ -29,7 +29,6 @@ import org.apache.ignite.ml.math.primitives.vector.Vector;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.trainers.DatasetTrainer;
import org.apache.ignite.ml.tree.boosting.GDBRegressionOnTreesTrainer;
-import org.apache.ignite.thread.IgniteThread;
import org.jetbrains.annotations.NotNull;
/**
@@ -52,43 +51,35 @@ public class GDBOnTreesRegressionTrainerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- GDBOnTreesRegressionTrainerExample.class.getSimpleName(), () -> {
+ // Create cache with training data.
+ CacheConfiguration<Integer, double[]> trainingSetCfg = createCacheConfiguration();
+ IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
- // Create cache with training data.
- CacheConfiguration<Integer, double[]> trainingSetCfg = createCacheConfiguration();
- IgniteCache<Integer, double[]> trainingSet = fillTrainingData(ignite, trainingSetCfg);
+ // Create regression trainer.
+ DatasetTrainer<ModelsComposition, Double> trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 1, 0.)
+ .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.001));
- // Create regression trainer.
- DatasetTrainer<ModelsComposition, Double> trainer = new GDBRegressionOnTreesTrainer(1.0, 2000, 1, 0.)
- .withCheckConvergenceStgyFactory(new MeanAbsValueConvergenceCheckerFactory(0.001));
+ // Train decision tree model.
+ Model<Vector, Double> mdl = trainer.fit(
+ ignite,
+ trainingSet,
+ (k, v) -> VectorUtils.of(v[0]),
+ (k, v) -> v[1]
+ );
- // Train decision tree model.
- Model<Vector, Double> mdl = trainer.fit(
- ignite,
- trainingSet,
- (k, v) -> VectorUtils.of(v[0]),
- (k, v) -> v[1]
- );
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Valid answer \t|");
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Valid answer \t|");
- System.out.println(">>> ---------------------------------");
+ // Calculate score.
+ for (int x = -5; x < 5; x++) {
+ double predicted = mdl.apply(VectorUtils.of(x));
- // Calculate score.
- for (int x = -5; x < 5; x++) {
- double predicted = mdl.apply(VectorUtils.of(x));
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.pow(x, 2));
+ }
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", predicted, Math.pow(x, 2));
- }
-
- System.out.println(">>> ---------------------------------");
-
- System.out.println(">>> GDB regression trainer example completed.");
- });
-
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> GDB regression trainer example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
index 4693744..ea235ee 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestClassificationExample.java
@@ -33,7 +33,6 @@ import org.apache.ignite.ml.dataset.feature.FeatureMeta;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.tree.randomforest.RandomForestClassifierTrainer;
import org.apache.ignite.ml.tree.randomforest.data.FeaturesCountSelectionStrategies;
-import org.apache.ignite.thread.IgniteThread;
/**
* Example represents a solution for the task of wine classification based on a
@@ -62,57 +61,50 @@ public class RandomForestClassificationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- RandomForestClassificationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- AtomicInteger idx = new AtomicInteger(0);
- RandomForestClassifierTrainer classifier = new RandomForestClassifierTrainer(
- IntStream.range(0, data[0].length - 1).mapToObj(
- x -> new FeatureMeta("", idx.getAndIncrement(), false)).collect(Collectors.toList())
- ).withAmountOfTrees(101)
- .withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
- .withMaxDepth(4)
- .withMinImpurityDelta(0.)
- .withSubSampleSize(0.3)
- .withSeed(0);
+ AtomicInteger idx = new AtomicInteger(0);
+ RandomForestClassifierTrainer classifier = new RandomForestClassifierTrainer(
+ IntStream.range(0, data[0].length - 1).mapToObj(
+ x -> new FeatureMeta("", idx.getAndIncrement(), false)).collect(Collectors.toList())
+ ).withAmountOfTrees(101)
+ .withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
+ .withMaxDepth(4)
+ .withMinImpurityDelta(0.)
+ .withSubSampleSize(0.3)
+ .withSeed(0);
- System.out.println(">>> Configured trainer: " + classifier.getClass().getSimpleName());
+ System.out.println(">>> Configured trainer: " + classifier.getClass().getSimpleName());
- ModelsComposition randomForest = classifier.fit(ignite, dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ ModelsComposition randomForest = classifier.fit(ignite, dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> Trained model: " + randomForest.toString(true));
+ System.out.println(">>> Trained model: " + randomForest.toString(true));
- int amountOfErrors = 0;
- int totalAmount = 0;
+ int amountOfErrors = 0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- double prediction = randomForest.apply(VectorUtils.of(inputs));
+ double prediction = randomForest.apply(VectorUtils.of(inputs));
- totalAmount++;
- if (groundTruth != prediction)
- amountOfErrors++;
-
- }
-
- System.out.println("\n>>> Evaluated model on " + totalAmount + " data points.");
-
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
- System.out.println(">>> Random Forest multi-class classification algorithm over cached dataset usage example completed.");
+ totalAmount++;
+ if (groundTruth != prediction)
+ amountOfErrors++;
}
- });
- igniteThread.start();
- igniteThread.join();
+ System.out.println("\n>>> Evaluated model on " + totalAmount + " data points.");
+
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
+ System.out.println(">>> Random Forest multi-class classification algorithm over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
index ee0c1c2..9b4aece 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tree/randomforest/RandomForestRegressionExample.java
@@ -37,7 +37,6 @@ import org.apache.ignite.ml.environment.parallelism.ParallelismStrategy;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.tree.randomforest.RandomForestRegressionTrainer;
import org.apache.ignite.ml.tree.randomforest.data.FeaturesCountSelectionStrategies;
-import org.apache.ignite.thread.IgniteThread;
/**
* Example represents a solution for the task of price predictions for houses in Boston based on a
@@ -66,68 +65,62 @@ public class RandomForestRegressionExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- RandomForestRegressionExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- AtomicInteger idx = new AtomicInteger(0);
- RandomForestRegressionTrainer trainer = new RandomForestRegressionTrainer(
- IntStream.range(0, data[0].length - 1).mapToObj(
- x -> new FeatureMeta("", idx.getAndIncrement(), false)).collect(Collectors.toList())
- ).withAmountOfTrees(101)
- .withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
- .withMaxDepth(4)
- .withMinImpurityDelta(0.)
- .withSubSampleSize(0.3)
- .withSeed(0);
+ AtomicInteger idx = new AtomicInteger(0);
+ RandomForestRegressionTrainer trainer = new RandomForestRegressionTrainer(
+ IntStream.range(0, data[0].length - 1).mapToObj(
+ x -> new FeatureMeta("", idx.getAndIncrement(), false)).collect(Collectors.toList())
+ ).withAmountOfTrees(101)
+ .withFeaturesCountSelectionStrgy(FeaturesCountSelectionStrategies.ONE_THIRD)
+ .withMaxDepth(4)
+ .withMinImpurityDelta(0.)
+ .withSubSampleSize(0.3)
+ .withSeed(0);
- trainer.setEnvironment(LearningEnvironment.builder()
- .withParallelismStrategy(ParallelismStrategy.Type.ON_DEFAULT_POOL)
- .withLoggingFactory(ConsoleLogger.factory(MLLogger.VerboseLevel.LOW))
- .build()
- );
+ trainer.setEnvironment(LearningEnvironment.builder()
+ .withParallelismStrategy(ParallelismStrategy.Type.ON_DEFAULT_POOL)
+ .withLoggingFactory(ConsoleLogger.factory(MLLogger.VerboseLevel.LOW))
+ .build()
+ );
- System.out.println(">>> Configured trainer: " + trainer.getClass().getSimpleName());
+ System.out.println(">>> Configured trainer: " + trainer.getClass().getSimpleName());
- ModelsComposition randomForest = trainer.fit(ignite, dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
- (k, v) -> v[v.length - 1]
- );
+ ModelsComposition randomForest = trainer.fit(ignite, dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 0, v.length - 1)),
+ (k, v) -> v[v.length - 1]
+ );
- System.out.println(">>> Trained model: " + randomForest.toString(true));
+ System.out.println(">>> Trained model: " + randomForest.toString(true));
- double mse = 0.0;
- double mae = 0.0;
- int totalAmount = 0;
+ double mse = 0.0;
+ double mae = 0.0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 0, val.length - 1);
- double groundTruth = val[val.length - 1];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 0, val.length - 1);
+ double groundTruth = val[val.length - 1];
- double prediction = randomForest.apply(VectorUtils.of(inputs));
+ double prediction = randomForest.apply(VectorUtils.of(inputs));
- mse += Math.pow(prediction - groundTruth, 2.0);
- mae += Math.abs(prediction - groundTruth);
+ mse += Math.pow(prediction - groundTruth, 2.0);
+ mae += Math.abs(prediction - groundTruth);
- totalAmount++;
- }
-
- System.out.println("\n>>> Evaluated model on " + totalAmount + " data points.");
+ totalAmount++;
+ }
- mse = mse / totalAmount;
- System.out.println("\n>>> Mean squared error (MSE) " + mse);
+ System.out.println("\n>>> Evaluated model on " + totalAmount + " data points.");
- mae = mae / totalAmount;
- System.out.println("\n>>> Mean absolute error (MAE) " + mae);
+ mse = mse / totalAmount;
+ System.out.println("\n>>> Mean squared error (MSE) " + mse);
- System.out.println(">>> Random Forest regression algorithm over cached dataset usage example completed.");
- }
- });
+ mae = mae / totalAmount;
+ System.out.println("\n>>> Mean absolute error (MAE) " + mae);
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Random Forest regression algorithm over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
index 78ec9f5..264dbf4 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_1_Read_and_Learn.java
@@ -28,7 +28,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Usage of {@link DecisionTreeClassificationTrainer} to predict death in the disaster.
@@ -48,47 +47,41 @@ public class Step_1_Read_and_Learn {
System.out.println(">>> Tutorial step 1 (read and learn) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_1_Read_and_Learn.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- IgniteBiFunction<Integer, Object[], Vector> featureExtractor
- = (k, v) -> VectorUtils.of((double) v[0], (double) v[5], (double) v[6]);
+ IgniteBiFunction<Integer, Object[], Vector> featureExtractor
+ = (k, v) -> VectorUtils.of((double) v[0], (double) v[5], (double) v[6]);
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
- DecisionTreeNode mdl = trainer.fit(
- ignite,
- dataCache,
- featureExtractor, // "pclass", "sibsp", "parch"
- lbExtractor
- );
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ featureExtractor, // "pclass", "sibsp", "parch"
+ lbExtractor
+ );
- System.out.println("\n>>> Trained model: " + mdl);
+ System.out.println("\n>>> Trained model: " + mdl);
- double accuracy = Evaluator.evaluate(
- dataCache,
- mdl,
- featureExtractor,
- lbExtractor,
- new Accuracy<>()
- );
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ featureExtractor,
+ lbExtractor,
+ new Accuracy<>()
+ );
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 1 (read and learn) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Tutorial step 1 (read and learn) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
index f86e1b6..df73235 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_2_Imputing.java
@@ -29,7 +29,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Usage of {@link ImputerTrainer} to fill missed data ({@code Double.NaN}) values in the chosen columns.
@@ -50,54 +49,48 @@ public class Step_2_Imputing {
System.out.println(">>> Tutorial step 2 (imputing) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_2_Imputing.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- IgniteBiFunction<Integer, Object[], Vector> featureExtractor
- = (k, v) -> VectorUtils.of((double) v[0], (double) v[5], (double) v[6]);
+ IgniteBiFunction<Integer, Object[], Vector> featureExtractor
+ = (k, v) -> VectorUtils.of((double) v[0], (double) v[5], (double) v[6]);
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- featureExtractor // "pclass", "sibsp", "parch"
- );
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
-
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
- ignite,
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
dataCache,
- imputingPreprocessor,
- lbExtractor
+ featureExtractor // "pclass", "sibsp", "parch"
);
- System.out.println("\n>>> Trained model: " + mdl);
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
- double accuracy = Evaluator.evaluate(
- dataCache,
- mdl,
- imputingPreprocessor,
- lbExtractor,
- new Accuracy<>()
- );
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + mdl);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ imputingPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
- System.out.println(">>> Tutorial step 2 (imputing) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Tutorial step 2 (imputing) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
index 03ff527..463a6ba 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial.java
@@ -30,7 +30,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Let's add two categorial features "sex", "embarked" to predict more precisely than in {@link Step_1_Read_and_Learn}.
@@ -53,65 +52,58 @@ public class Step_3_Categorial {
System.out.println(">>> Tutorial step 3 (categorial) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_3_Categorial.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- // Defines first preprocessor that extracts features from an upstream data.
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[5], v[6], v[10]}; // "pclass", "sibsp", "parch", "sex", "embarked"
+ // Defines first preprocessor that extracts features from an upstream data.
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[5], v[6], v[10]}; // "pclass", "sibsp", "parch", "sex", "embarked"
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(4)
- .fit(ignite,
- dataCache,
- featureExtractor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
-
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
- ignite,
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(4)
+ .fit(ignite,
dataCache,
- imputingPreprocessor,
- lbExtractor
- );
+ featureExtractor
+ );
- System.out.println("\n>>> Trained model: " + mdl);
-
- double accuracy = Evaluator.evaluate(
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
dataCache,
- mdl,
- imputingPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ strEncoderPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 3 (categorial) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + mdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ imputingPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 3 (categorial) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
index a4535ba..93e7e79 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_3_Categorial_with_One_Hot_Encoder.java
@@ -30,7 +30,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Let's add two categorial features "sex", "embarked" to predict more precisely than in {@link Step_1_Read_and_Learn}..
@@ -54,68 +53,61 @@ public class Step_3_Categorial_with_One_Hot_Encoder {
System.out.println(">>> Tutorial step 3 (categorial with One-hot encoder) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_3_Categorial_with_One_Hot_Encoder.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[5], v[6], v[10]
- }; // "pclass", "sibsp", "parch", "sex", "embarked"
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- IgniteBiFunction<Integer, Object[], Vector> oneHotEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.ONE_HOT_ENCODER)
- .withEncodedFeature(0)
- .withEncodedFeature(1)
- .withEncodedFeature(4)
- .fit(ignite,
- dataCache,
- featureExtractor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- oneHotEncoderPreprocessor
- );
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+
+ // Defines first preprocessor that extracts features from an upstream data.
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[5], v[6], v[10]
+ }; // "pclass", "sibsp", "parch", "sex", "embarked"
+
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+
+ IgniteBiFunction<Integer, Object[], Vector> oneHotEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.ONE_HOT_ENCODER)
+ .withEncodedFeature(0)
+ .withEncodedFeature(1)
+ .withEncodedFeature(4)
+ .fit(ignite,
+ dataCache,
+ featureExtractor
+ );
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
- ignite,
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
dataCache,
- imputingPreprocessor,
- lbExtractor
+ oneHotEncoderPreprocessor
);
- System.out.println("\n>>> Trained model: " + mdl);
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
- double accuracy = Evaluator.evaluate(
- dataCache,
- mdl,
- imputingPreprocessor,
- lbExtractor,
- new Accuracy<>()
- );
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor,
+ lbExtractor
+ );
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
+ System.out.println("\n>>> Trained model: " + mdl);
- System.out.println(">>> Tutorial step 3 (categorial with One-hot encoder) example started.");
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ imputingPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
- igniteThread.start();
+ System.out.println(">>> Tutorial step 3 (categorial with One-hot encoder) example started.");
- igniteThread.join();
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
index 789d7e8..bbeedb6 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_4_Add_age_fare.java
@@ -30,7 +30,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Add yet two numerical features "age", "fare" to improve our model over {@link Step_3_Categorial}.
@@ -51,66 +50,59 @@ public class Step_4_Add_age_fare {
System.out.println(">>> Tutorial step 4 (add age and fare) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_4_Add_age_fare.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- featureExtractor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
-
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
- ignite,
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here.
+ .fit(ignite,
dataCache,
- imputingPreprocessor,
- lbExtractor
- );
+ featureExtractor
+ );
- System.out.println("\n>>> Trained model: " + mdl);
-
- double accuracy = Evaluator.evaluate(
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
dataCache,
- mdl,
- imputingPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ strEncoderPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 4 (add age and fare) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ imputingPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + mdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ imputingPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 4 (add age and fare) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
index e3de585..7d934d7 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* {@link MinMaxScalerTrainer} and {@link NormalizationTrainer} are used in this example due to different values
@@ -54,80 +53,74 @@ public class Step_5_Scaling {
System.out.println(">>> Tutorial step 5 (scaling) example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_5_Scaling.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
-
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
-
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
-
- IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6) // <--- Changed index here.
- .fit(ignite,
- dataCache,
- featureExtractor
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
+
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+
+ IgniteBiFunction<Integer, Object[], Vector> strEncoderPreprocessor = new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6) // <--- Changed index here.
+ .fit(ignite,
+ dataCache,
+ featureExtractor
+ );
+
+ IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
+ .fit(ignite,
+ dataCache,
+ strEncoderPreprocessor
);
- IgniteBiFunction<Integer, Object[], Vector> imputingPreprocessor = new ImputerTrainer<Integer, Object[]>()
- .fit(ignite,
- dataCache,
- strEncoderPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
- .fit(
- ignite,
- dataCache,
- imputingPreprocessor
- );
-
- IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
- .withP(1)
- .fit(
- ignite,
- dataCache,
- minMaxScalerPreprocessor
- );
-
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
-
- // Train decision tree model.
- DecisionTreeNode mdl = trainer.fit(
+ IgniteBiFunction<Integer, Object[], Vector> minMaxScalerPreprocessor = new MinMaxScalerTrainer<Integer, Object[]>()
+ .fit(
ignite,
dataCache,
- normalizationPreprocessor,
- lbExtractor
+ imputingPreprocessor
);
- System.out.println("\n>>> Trained model: " + mdl);
-
- double accuracy = Evaluator.evaluate(
+ IgniteBiFunction<Integer, Object[], Vector> normalizationPreprocessor = new NormalizationTrainer<Integer, Object[]>()
+ .withP(1)
+ .fit(
+ ignite,
dataCache,
- mdl,
- normalizationPreprocessor,
- lbExtractor,
- new Accuracy<>()
+ minMaxScalerPreprocessor
);
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
-
- System.out.println(">>> Tutorial step 5 (scaling) example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
- igniteThread.join();
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(5, 0);
+
+ // Train decision tree model.
+ DecisionTreeNode mdl = trainer.fit(
+ ignite,
+ dataCache,
+ normalizationPreprocessor,
+ lbExtractor
+ );
+
+ System.out.println("\n>>> Trained model: " + mdl);
+
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ normalizationPreprocessor,
+ lbExtractor,
+ new Accuracy<>()
+ );
+
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
+
+ System.out.println(">>> Tutorial step 5 (scaling) example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
index 1d5900f..cc0a278 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/tutorial/Step_5_Scaling_with_Pipeline.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.preprocessing.normalization.NormalizationTrainer;
import org.apache.ignite.ml.selection.scoring.evaluator.Evaluator;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* {@link MinMaxScalerTrainer} and {@link NormalizationTrainer} are used in this example due to different values
@@ -54,54 +53,48 @@ public class Step_5_Scaling_with_Pipeline {
System.out.println(">>> Tutorial step 5 (scaling) via Pipeline example started.");
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- Step_5_Scaling_with_Pipeline.class.getSimpleName(), () -> {
- try {
- IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
+ try {
+ IgniteCache<Integer, Object[]> dataCache = TitanicUtils.readPassengers(ignite);
- // Defines first preprocessor that extracts features from an upstream data.
- // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
- IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
- = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
+ // Defines first preprocessor that extracts features from an upstream data.
+ // Extracts "pclass", "sibsp", "parch", "sex", "embarked", "age", "fare".
+ IgniteBiFunction<Integer, Object[], Object[]> featureExtractor
+ = (k, v) -> new Object[]{v[0], v[3], v[4], v[5], v[6], v[8], v[10]};
- IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
+ IgniteBiFunction<Integer, Object[], Double> lbExtractor = (k, v) -> (double) v[1];
- PipelineMdl<Integer, Object[]> mdl = new Pipeline<Integer, Object[], Object[]>()
- .addFeatureExtractor(featureExtractor)
- .addLabelExtractor(lbExtractor)
- .addPreprocessor(new EncoderTrainer<Integer, Object[]>()
- .withEncoderType(EncoderType.STRING_ENCODER)
- .withEncodedFeature(1)
- .withEncodedFeature(6))
- .addPreprocessor(new ImputerTrainer<Integer, Object[]>())
- .addPreprocessor(new MinMaxScalerTrainer<Integer, Object[]>())
- .addPreprocessor(new NormalizationTrainer<Integer, Object[]>()
- .withP(1))
- .addTrainer(new DecisionTreeClassificationTrainer(5, 0))
- .fit(ignite, dataCache);
+ PipelineMdl<Integer, Object[]> mdl = new Pipeline<Integer, Object[], Object[]>()
+ .addFeatureExtractor(featureExtractor)
+ .addLabelExtractor(lbExtractor)
+ .addPreprocessor(new EncoderTrainer<Integer, Object[]>()
+ .withEncoderType(EncoderType.STRING_ENCODER)
+ .withEncodedFeature(1)
+ .withEncodedFeature(6))
+ .addPreprocessor(new ImputerTrainer<Integer, Object[]>())
+ .addPreprocessor(new MinMaxScalerTrainer<Integer, Object[]>())
+ .addPreprocessor(new NormalizationTrainer<Integer, Object[]>()
+ .withP(1))
+ .addTrainer(new DecisionTreeClassificationTrainer(5, 0))
+ .fit(ignite, dataCache);
- System.out.println("\n>>> Trained model: " + mdl);
+ System.out.println("\n>>> Trained model: " + mdl);
- double accuracy = Evaluator.evaluate(
- dataCache,
- mdl,
- mdl.getFeatureExtractor(),
- mdl.getLabelExtractor(),
- new Accuracy<>()
- );
+ double accuracy = Evaluator.evaluate(
+ dataCache,
+ mdl,
+ mdl.getFeatureExtractor(),
+ mdl.getLabelExtractor(),
+ new Accuracy<>()
+ );
- System.out.println("\n>>> Accuracy " + accuracy);
- System.out.println("\n>>> Test Error " + (1 - accuracy));
+ System.out.println("\n>>> Accuracy " + accuracy);
+ System.out.println("\n>>> Test Error " + (1 - accuracy));
- System.out.println(">>> Tutorial step 5 (scaling) via Pipeline example completed.");
- }
- catch (FileNotFoundException e) {
- e.printStackTrace();
- }
- });
-
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Tutorial step 5 (scaling) via Pipeline example completed.");
+ }
+ catch (FileNotFoundException e) {
+ e.printStackTrace();
+ }
}
}
}
[03/21] ignite git commit: IGNITE-9253: Rename
activation/deactivation commands for REST API. - Fixes #4525.
Posted by sb...@apache.org.
IGNITE-9253: Rename activation/deactivation commands for REST API. - Fixes #4525.
Signed-off-by: shroman <rs...@yahoo.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/c7227cf1
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/c7227cf1
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/c7227cf1
Branch: refs/heads/ignite-gg-14206
Commit: c7227cf117b826fff013f757531c2737d6fc94de
Parents: 5aef881
Author: shroman <rs...@yahoo.com>
Authored: Fri Sep 28 17:56:06 2018 +0900
Committer: shroman <rs...@yahoo.com>
Committed: Fri Sep 28 17:56:06 2018 +0900
----------------------------------------------------------------------
.../JettyRestProcessorAbstractSelfTest.java | 21 ++++++++-------
.../processors/rest/GridRestCommand.java | 12 +++++++--
.../processors/rest/GridRestProcessor.java | 2 ++
.../cluster/GridChangeStateCommandHandler.java | 25 +++++++++++------
.../protocols/tcp/GridTcpRestNioListener.java | 6 ++---
.../http/jetty/GridJettyRestHandler.java | 28 +++++++++++---------
6 files changed, 60 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7227cf1/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
index f39c280..a972bc3 100644
--- a/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
+++ b/modules/clients/src/test/java/org/apache/ignite/internal/processors/rest/JettyRestProcessorAbstractSelfTest.java
@@ -850,8 +850,12 @@ public abstract class JettyRestProcessorAbstractSelfTest extends JettyRestProces
public void testDeactivateActivate() throws Exception {
assertClusterState(true);
- changeClusterState(false);
- changeClusterState(true);
+ changeClusterState(GridRestCommand.CLUSTER_DEACTIVATE);
+ changeClusterState(GridRestCommand.CLUSTER_ACTIVATE);
+
+ // same for deprecated.
+ changeClusterState(GridRestCommand.CLUSTER_INACTIVE);
+ changeClusterState(GridRestCommand.CLUSTER_ACTIVE);
initCache();
}
@@ -2797,7 +2801,7 @@ public abstract class JettyRestProcessorAbstractSelfTest extends JettyRestProces
* @return This helper for chaining method calls.
*/
public VisorGatewayArgument forNode(ClusterNode node) {
- put("p1", node != null ? node.id().toString() : null);
+ put("p1", node != null ? node.id().toString() : null);
return this;
}
@@ -2991,18 +2995,17 @@ public abstract class JettyRestProcessorAbstractSelfTest extends JettyRestProces
/**
* Change cluster state and test new state.
*
- * @param state Desired state.
+ * @param cmd Command.
* @throws Exception If failed.
*/
- private void changeClusterState(boolean state) throws Exception {
- GridRestCommand cmd = state ? GridRestCommand.CLUSTER_ACTIVE : GridRestCommand.CLUSTER_INACTIVE;
-
+ private void changeClusterState(GridRestCommand cmd) throws Exception {
String ret = content(null, cmd);
JsonNode res = jsonResponse(ret);
- assertTrue(res.isNull());
+ assertFalse(res.isNull());
+ assertTrue(res.asText().startsWith(cmd.key()));
- assertClusterState(state);
+ assertClusterState(cmd == GridRestCommand.CLUSTER_ACTIVATE || cmd == GridRestCommand.CLUSTER_ACTIVE);
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7227cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
index 265fe40..587ed2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestCommand.java
@@ -159,13 +159,21 @@ public enum GridRestCommand {
/** Close query. */
CLOSE_SQL_QUERY("qrycls"),
- /** */
+ /** @deprecated Use {@link #CLUSTER_ACTIVATE} instead. */
+ @Deprecated
CLUSTER_ACTIVE("active"),
- /** */
+ /** @deprecated Use {@link #CLUSTER_DEACTIVATE} instead. */
+ @Deprecated
CLUSTER_INACTIVE("inactive"),
/** */
+ CLUSTER_ACTIVATE("activate"),
+
+ /** */
+ CLUSTER_DEACTIVATE("deactivate"),
+
+ /** */
CLUSTER_CURRENT_STATE("currentstate"),
/** */
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7227cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
index b6c1310..30d2f0a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/GridRestProcessor.java
@@ -890,6 +890,8 @@ public class GridRestProcessor extends GridProcessorAdapter {
case CLUSTER_ACTIVE:
case CLUSTER_INACTIVE:
+ case CLUSTER_ACTIVATE:
+ case CLUSTER_DEACTIVATE:
perm = SecurityPermission.ADMIN_OPS;
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7227cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
index 619be34..c9b5a12 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/handlers/cluster/GridChangeStateCommandHandler.java
@@ -18,7 +18,6 @@
package org.apache.ignite.internal.processors.rest.handlers.cluster;
import java.util.Collection;
-
import org.apache.ignite.internal.GridKernalContext;
import org.apache.ignite.internal.IgniteInternalFuture;
import org.apache.ignite.internal.processors.rest.GridRestCommand;
@@ -31,8 +30,10 @@ import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVATE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_CURRENT_STATE;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_DEACTIVATE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_INACTIVE;
/**
@@ -41,7 +42,7 @@ import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER
public class GridChangeStateCommandHandler extends GridRestCommandHandlerAdapter {
/** Commands. */
private static final Collection<GridRestCommand> commands =
- U.sealList(CLUSTER_ACTIVE, CLUSTER_INACTIVE, CLUSTER_CURRENT_STATE);
+ U.sealList(CLUSTER_ACTIVATE, CLUSTER_DEACTIVATE, CLUSTER_CURRENT_STATE, CLUSTER_ACTIVE, CLUSTER_INACTIVE);
/**
* @param ctx Context.
@@ -64,13 +65,21 @@ public class GridChangeStateCommandHandler extends GridRestCommandHandlerAdapter
final GridRestResponse res = new GridRestResponse();
try {
- if (req.command().equals(CLUSTER_CURRENT_STATE)) {
- Boolean currentState = ctx.state().publicApiActiveState(false);
-
- res.setResponse(currentState);
+ switch (req.command()) {
+ case CLUSTER_CURRENT_STATE:
+ Boolean currentState = ctx.state().publicApiActiveState(false);
+
+ res.setResponse(currentState);
+ break;
+ case CLUSTER_ACTIVE:
+ case CLUSTER_INACTIVE:
+ log.warning(req.command().key() + " is deprecated. Use newer commands.");
+ default:
+ ctx.grid().cluster().active(req.active());
+
+ res.setResponse(req.command().key() + " started");
+ break;
}
- else
- ctx.grid().active(req.active());
fut.onDone(res);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7227cf1/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
index 23c25c7..d4d2b07 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/rest/protocols/tcp/GridTcpRestNioListener.java
@@ -72,8 +72,8 @@ import static org.apache.ignite.internal.processors.rest.GridRestCommand.EXE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.NODE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.NOOP;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.TOPOLOGY;
-import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVE;
-import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_INACTIVE;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVATE;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_DEACTIVATE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_CURRENT_STATE;
import static org.apache.ignite.internal.processors.rest.client.message.GridClientCacheRequest.GridCacheOperation.APPEND;
import static org.apache.ignite.internal.processors.rest.client.message.GridClientCacheRequest.GridCacheOperation.CAS;
@@ -366,7 +366,7 @@ public class GridTcpRestNioListener extends GridNioServerListenerAdapter<GridCli
}
else {
restChangeReq.active(req.active());
- restChangeReq.command(req.active() ? CLUSTER_ACTIVE : CLUSTER_INACTIVE);
+ restChangeReq.command(req.active() ? CLUSTER_ACTIVATE : CLUSTER_DEACTIVATE);
}
restReq = restChangeReq;
http://git-wip-us.apache.org/repos/asf/ignite/blob/c7227cf1/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
----------------------------------------------------------------------
diff --git a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
index 12dc0dd..03bed69 100644
--- a/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
+++ b/modules/rest-http/src/main/java/org/apache/ignite/internal/processors/rest/protocols/http/jetty/GridJettyRestHandler.java
@@ -17,6 +17,8 @@
package org.apache.ignite.internal.processors.rest.protocols.http.jetty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
@@ -40,7 +42,6 @@ import javax.servlet.ServletOutputStream;
import javax.servlet.ServletRequest;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteLogger;
import org.apache.ignite.IgniteSystemProperties;
@@ -64,11 +65,8 @@ import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteClosure;
import org.apache.ignite.lang.IgniteUuid;
import org.apache.ignite.plugin.security.SecurityCredentials;
-
import org.eclipse.jetty.server.Request;
import org.eclipse.jetty.server.handler.AbstractHandler;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.IgniteSystemProperties.IGNITE_REST_GETALL_AS_ARRAY;
@@ -77,9 +75,10 @@ import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_C
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_GET_ALL;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_PUT_ALL;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CACHE_REMOVE_ALL;
-import static org.apache.ignite.internal.processors.rest.GridRestCommand.EXECUTE_SQL_QUERY;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVATE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_ACTIVE;
import static org.apache.ignite.internal.processors.rest.GridRestCommand.CLUSTER_CURRENT_STATE;
+import static org.apache.ignite.internal.processors.rest.GridRestCommand.EXECUTE_SQL_QUERY;
import static org.apache.ignite.internal.processors.rest.GridRestResponse.STATUS_FAILED;
/**
@@ -110,10 +109,10 @@ public class GridJettyRestHandler extends AbstractHandler {
/** */
private static final String WRITE_SYNCHRONIZATION_MODE_PARAM = "writeSynchronizationMode";
- /**@deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */
+ /** @deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */
private static final String IGNITE_LOGIN = "ignite.login";
- /**@deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */
+ /** @deprecated Should be replaced with AUTHENTICATION + token in IGNITE 3.0 */
private static final String IGNITE_PASSWORD = "ignite.password";
/** */
@@ -425,7 +424,7 @@ public class GridJettyRestHandler extends AbstractHandler {
cmdRes = new GridRestResponse(STATUS_FAILED, e.getMessage());
}
- try(ServletOutputStream os = res.getOutputStream()) {
+ try (ServletOutputStream os = res.getOutputStream()) {
try {
// Try serialize.
jsonMapper.writeValue(NULL_OUTPUT_STREAM, cmdRes);
@@ -740,13 +739,17 @@ public class GridJettyRestHandler extends AbstractHandler {
case CLUSTER_ACTIVE:
case CLUSTER_INACTIVE:
+ case CLUSTER_ACTIVATE:
+ case CLUSTER_DEACTIVATE:
case CLUSTER_CURRENT_STATE: {
GridRestChangeStateRequest restReq0 = new GridRestChangeStateRequest();
if (cmd == CLUSTER_CURRENT_STATE)
restReq0.reqCurrentState();
+ else if (cmd == CLUSTER_ACTIVE || cmd == CLUSTER_ACTIVATE)
+ restReq0.active(true);
else
- restReq0.active(cmd == CLUSTER_ACTIVE);
+ restReq0.active(false);
restReq = restReq0;
@@ -911,14 +914,14 @@ public class GridJettyRestHandler extends AbstractHandler {
}
/**
- *
* @param params Parameters.
* @param userParam Parameter name to take user name.
* @param pwdParam Parameter name to take password.
* @param restReq Request to add credentials if any.
* @return {@code true} If params contains credentials.
*/
- private boolean credentials(Map<String, Object> params, String userParam, String pwdParam, GridRestRequest restReq) {
+ private boolean credentials(Map<String, Object> params, String userParam, String pwdParam,
+ GridRestRequest restReq) {
boolean hasCreds = params.containsKey(userParam) || params.containsKey(pwdParam);
if (hasCreds) {
@@ -939,7 +942,8 @@ public class GridJettyRestHandler extends AbstractHandler {
* @param params Parameters map.
* @return Values.
*/
- protected List<Object> values(String type, String keyPrefix, Map<String, Object> params) throws IgniteCheckedException {
+ protected List<Object> values(String type, String keyPrefix,
+ Map<String, Object> params) throws IgniteCheckedException {
assert keyPrefix != null;
List<Object> vals = new LinkedList<>();
[06/21] ignite git commit: IGNITE-9711: [ML] Remove IgniteThread
wrapper from ml examples
Posted by sb...@apache.org.
IGNITE-9711: [ML] Remove IgniteThread wrapper from ml examples
this closes #4849
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/609266fe
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/609266fe
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/609266fe
Branch: refs/heads/ignite-gg-14206
Commit: 609266fe2797c07599a893625f933740a25d049d
Parents: c7227cf
Author: YuriBabak <y....@gmail.com>
Authored: Fri Sep 28 11:57:58 2018 +0300
Committer: Yury Babak <yb...@gridgain.com>
Committed: Fri Sep 28 11:57:58 2018 +0300
----------------------------------------------------------------------
.../clustering/KMeansClusterizationExample.java | 75 +++--
.../ml/knn/ANNClassificationExample.java | 97 +++---
.../ml/knn/KNNClassificationExample.java | 69 ++---
.../examples/ml/knn/KNNRegressionExample.java | 79 +++--
.../examples/ml/nn/MLPTrainerExample.java | 130 ++++----
.../LinearRegressionLSQRTrainerExample.java | 56 ++--
...ssionLSQRTrainerWithMinMaxScalerExample.java | 69 ++---
.../LinearRegressionSGDTrainerExample.java | 65 ++--
.../LogisticRegressionSGDTrainerExample.java | 84 +++---
...gressionMultiClassClassificationExample.java | 169 +++++------
.../ml/selection/cv/CrossValidationExample.java | 58 ++--
.../split/TrainTestDatasetSplitterExample.java | 69 ++---
.../binary/SVMBinaryClassificationExample.java | 79 +++--
.../SVMMultiClassClassificationExample.java | 151 +++++-----
...ecisionTreeClassificationTrainerExample.java | 74 ++---
.../DecisionTreeRegressionTrainerExample.java | 63 ++--
.../GDBOnTreesClassificationTrainerExample.java | 58 ++--
.../GDBOnTreesRegressionTrainerExample.java | 55 ++--
.../RandomForestClassificationExample.java | 76 +++--
.../RandomForestRegressionExample.java | 91 +++---
.../ml/tutorial/Step_1_Read_and_Learn.java | 61 ++--
.../examples/ml/tutorial/Step_2_Imputing.java | 71 ++---
.../examples/ml/tutorial/Step_3_Categorial.java | 96 +++---
.../Step_3_Categorial_with_One_Hot_Encoder.java | 98 +++---
.../ml/tutorial/Step_4_Add_age_fare.java | 98 +++---
.../examples/ml/tutorial/Step_5_Scaling.java | 125 ++++----
.../tutorial/Step_5_Scaling_with_Pipeline.java | 77 +++--
.../ignite/examples/ml/tutorial/Step_6_KNN.java | 127 ++++----
.../ml/tutorial/Step_7_Split_train_test.java | 136 ++++-----
.../ignite/examples/ml/tutorial/Step_8_CV.java | 218 +++++++-------
.../ml/tutorial/Step_8_CV_with_Param_Grid.java | 200 ++++++-------
.../ml/tutorial/Step_9_Go_to_LogReg.java | 296 +++++++++----------
32 files changed, 1507 insertions(+), 1763 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
index 152375a..567775b 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/clustering/KMeansClusterizationExample.java
@@ -30,7 +30,6 @@ import org.apache.ignite.ml.clustering.kmeans.KMeansTrainer;
import org.apache.ignite.ml.math.Tracer;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run KMeans clustering algorithm ({@link KMeansTrainer}) over distributed dataset.
@@ -55,58 +54,52 @@ public class KMeansClusterizationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- KMeansClusterizationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- KMeansTrainer trainer = new KMeansTrainer()
- .withSeed(7867L);
+ KMeansTrainer trainer = new KMeansTrainer()
+ .withSeed(7867L);
- KMeansModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ KMeansModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> KMeans centroids");
- Tracer.showAscii(mdl.getCenters()[0]);
- Tracer.showAscii(mdl.getCenters()[1]);
- System.out.println(">>>");
+ System.out.println(">>> KMeans centroids");
+ Tracer.showAscii(mdl.getCenters()[0]);
+ Tracer.showAscii(mdl.getCenters()[1]);
+ System.out.println(">>>");
- System.out.println(">>> -----------------------------------");
- System.out.println(">>> | Predicted cluster\t| Real Label\t|");
- System.out.println(">>> -----------------------------------");
+ System.out.println(">>> -----------------------------------");
+ System.out.println(">>> | Predicted cluster\t| Real Label\t|");
+ System.out.println(">>> -----------------------------------");
- int amountOfErrors = 0;
- int totalAmount = 0;
+ int amountOfErrors = 0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- double prediction = mdl.apply(new DenseVector(inputs));
+ double prediction = mdl.apply(new DenseVector(inputs));
- totalAmount++;
- if (groundTruth != prediction)
- amountOfErrors++;
+ totalAmount++;
+ if (groundTruth != prediction)
+ amountOfErrors++;
- System.out.printf(">>> | %.4f\t\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
-
- System.out.println(">>> ---------------------------------");
+ System.out.printf(">>> | %.4f\t\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> KMeans clustering algorithm over cached dataset usage example completed.");
- }
- });
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> KMeans clustering algorithm over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/knn/ANNClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/ANNClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/ANNClassificationExample.java
index 8a2d786..c9490fc 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/ANNClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/ANNClassificationExample.java
@@ -34,7 +34,6 @@ import org.apache.ignite.ml.math.distances.EuclideanDistance;
import org.apache.ignite.ml.math.distances.ManhattanDistance;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run ANN multi-class classification trainer ({@link ANNClassificationTrainer}) over distributed dataset.
@@ -59,73 +58,67 @@ public class ANNClassificationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- ANNClassificationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
+ IgniteCache<Integer, double[]> dataCache = getTestCache(ignite);
- ANNClassificationTrainer trainer = new ANNClassificationTrainer()
- .withDistance(new ManhattanDistance())
- .withK(50)
- .withMaxIterations(1000)
- .withSeed(1234L)
- .withEpsilon(1e-2);
+ ANNClassificationTrainer trainer = new ANNClassificationTrainer()
+ .withDistance(new ManhattanDistance())
+ .withK(50)
+ .withMaxIterations(1000)
+ .withSeed(1234L)
+ .withEpsilon(1e-2);
- long startTrainingTime = System.currentTimeMillis();
+ long startTrainingTime = System.currentTimeMillis();
- NNClassificationModel knnMdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- ).withK(5)
- .withDistanceMeasure(new EuclideanDistance())
- .withStrategy(NNStrategy.WEIGHTED);
+ NNClassificationModel knnMdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ ).withK(5)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.WEIGHTED);
- long endTrainingTime = System.currentTimeMillis();
+ long endTrainingTime = System.currentTimeMillis();
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
-
- int amountOfErrors = 0;
- int totalAmount = 0;
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- long totalPredictionTime = 0L;
+ int amountOfErrors = 0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ long totalPredictionTime = 0L;
- long startPredictionTime = System.currentTimeMillis();
- double prediction = knnMdl.apply(new DenseVector(inputs));
- long endPredictionTime = System.currentTimeMillis();
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- totalPredictionTime += (endPredictionTime - startPredictionTime);
+ long startPredictionTime = System.currentTimeMillis();
+ double prediction = knnMdl.apply(new DenseVector(inputs));
+ long endPredictionTime = System.currentTimeMillis();
- totalAmount++;
- if (groundTruth != prediction)
- amountOfErrors++;
+ totalPredictionTime += (endPredictionTime - startPredictionTime);
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ totalAmount++;
+ if (groundTruth != prediction)
+ amountOfErrors++;
- System.out.println(">>> ---------------------------------");
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
- System.out.println("Training costs = " + (endTrainingTime - startTrainingTime));
- System.out.println("Prediction costs = " + totalPredictionTime);
+ System.out.println(">>> ---------------------------------");
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
- System.out.println(totalAmount);
+ System.out.println("Training costs = " + (endTrainingTime - startTrainingTime));
+ System.out.println("Prediction costs = " + totalPredictionTime);
- System.out.println(">>> ANN multi-class classification algorithm over cached dataset usage example completed.");
- }
- });
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
+ System.out.println(totalAmount);
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> ANN multi-class classification algorithm over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
index cf285a4..5cbb2ad 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNClassificationExample.java
@@ -31,7 +31,6 @@ import org.apache.ignite.ml.knn.classification.NNStrategy;
import org.apache.ignite.ml.math.distances.EuclideanDistance;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run kNN multi-class classification trainer ({@link KNNClassificationTrainer}) over distributed dataset.
@@ -56,54 +55,48 @@ public class KNNClassificationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- KNNClassificationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- KNNClassificationTrainer trainer = new KNNClassificationTrainer();
+ KNNClassificationTrainer trainer = new KNNClassificationTrainer();
- NNClassificationModel knnMdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- ).withK(3)
- .withDistanceMeasure(new EuclideanDistance())
- .withStrategy(NNStrategy.WEIGHTED);
+ NNClassificationModel knnMdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ ).withK(3)
+ .withDistanceMeasure(new EuclideanDistance())
+ .withStrategy(NNStrategy.WEIGHTED);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
-
- int amountOfErrors = 0;
- int totalAmount = 0;
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ int amountOfErrors = 0;
+ int totalAmount = 0;
- double prediction = knnMdl.apply(new DenseVector(inputs));
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- totalAmount++;
- if (groundTruth != prediction)
- amountOfErrors++;
+ double prediction = knnMdl.apply(new DenseVector(inputs));
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ totalAmount++;
+ if (groundTruth != prediction)
+ amountOfErrors++;
- System.out.println(">>> ---------------------------------");
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> kNN multi-class classification algorithm over cached dataset usage example completed.");
- }
- });
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double) totalAmount));
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> kNN multi-class classification algorithm over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
index 78f38c8..3969f0c 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/knn/KNNRegressionExample.java
@@ -31,7 +31,6 @@ import org.apache.ignite.ml.knn.regression.KNNRegressionTrainer;
import org.apache.ignite.ml.math.distances.ManhattanDistance;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run kNN regression trainer ({@link KNNRegressionTrainer}) over distributed dataset.
@@ -57,61 +56,55 @@ public class KNNRegressionExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- KNNRegressionExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- KNNRegressionTrainer trainer = new KNNRegressionTrainer();
+ KNNRegressionTrainer trainer = new KNNRegressionTrainer();
- KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- ).withK(5)
- .withDistanceMeasure(new ManhattanDistance())
- .withStrategy(NNStrategy.WEIGHTED);
+ KNNRegressionModel knnMdl = (KNNRegressionModel) trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ ).withK(5)
+ .withDistanceMeasure(new ManhattanDistance())
+ .withStrategy(NNStrategy.WEIGHTED);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
-
- int totalAmount = 0;
- // Calculate mean squared error (MSE)
- double mse = 0.0;
- // Calculate mean absolute error (MAE)
- double mae = 0.0;
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ int totalAmount = 0;
+ // Calculate mean squared error (MSE)
+ double mse = 0.0;
+ // Calculate mean absolute error (MAE)
+ double mae = 0.0;
- double prediction = knnMdl.apply(new DenseVector(inputs));
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- mse += Math.pow(prediction - groundTruth, 2.0);
- mae += Math.abs(prediction - groundTruth);
+ double prediction = knnMdl.apply(new DenseVector(inputs));
- totalAmount++;
+ mse += Math.pow(prediction - groundTruth, 2.0);
+ mae += Math.abs(prediction - groundTruth);
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ totalAmount++;
- System.out.println(">>> ---------------------------------");
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
+ }
- mse = mse / totalAmount;
- System.out.println("\n>>> Mean squared error (MSE) " + mse);
+ System.out.println(">>> ---------------------------------");
- mae = mae / totalAmount;
- System.out.println("\n>>> Mean absolute error (MAE) " + mae);
+ mse = mse / totalAmount;
+ System.out.println("\n>>> Mean squared error (MSE) " + mse);
- System.out.println(">>> kNN regression over cached dataset usage example completed.");
- }
- });
+ mae = mae / totalAmount;
+ System.out.println("\n>>> Mean absolute error (MAE) " + mae);
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> kNN regression over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
index 3e5a98c..6d5745e 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/nn/MLPTrainerExample.java
@@ -34,7 +34,6 @@ import org.apache.ignite.ml.nn.architecture.MLPArchitecture;
import org.apache.ignite.ml.optimization.LossFunctions;
import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpdate;
import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
-import org.apache.ignite.thread.IgniteThread;
/**
* Example of using distributed {@link MultilayerPerceptron}.
@@ -70,76 +69,65 @@ public class MLPTrainerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- // Create IgniteThread, we must work with SparseDistributedMatrix inside IgniteThread
- // because we create ignite cache internally.
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- MLPTrainerExample.class.getSimpleName(), () -> {
-
- // Create cache with training data.
- CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
- trainingSetCfg.setName("TRAINING_SET");
- trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
-
- IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
-
- // Fill cache with training data.
- trainingSet.put(0, new LabeledPoint(0, 0, 0));
- trainingSet.put(1, new LabeledPoint(0, 1, 1));
- trainingSet.put(2, new LabeledPoint(1, 0, 1));
- trainingSet.put(3, new LabeledPoint(1, 1, 0));
-
- // Define a layered architecture.
- MLPArchitecture arch = new MLPArchitecture(2).
- withAddedLayer(10, true, Activators.RELU).
- withAddedLayer(1, false, Activators.SIGMOID);
-
- // Define a neural network trainer.
- MLPTrainer<SimpleGDParameterUpdate> trainer = new MLPTrainer<>(
- arch,
- LossFunctions.MSE,
- new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(0.1),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ),
- 3000,
- 4,
- 50,
- 123L
- );
-
- // Train neural network and get multilayer perceptron model.
- MultilayerPerceptron mlp = trainer.fit(
- ignite,
- trainingSet,
- (k, v) -> VectorUtils.of(v.x, v.y),
- (k, v) -> new double[] {v.lb}
- );
-
- int totalCnt = 4;
- int failCnt = 0;
-
- // Calculate score.
- for (int i = 0; i < 4; i++) {
- LabeledPoint pnt = trainingSet.get(i);
- Matrix predicted = mlp.apply(new DenseMatrix(new double[][] {{pnt.x, pnt.y}}));
-
- double predictedVal = predicted.get(0, 0);
- double lbl = pnt.lb;
- System.out.printf(">>> key: %d\t\t predicted: %.4f\t\tlabel: %.4f\n", i, predictedVal, lbl);
- failCnt += Math.abs(predictedVal - lbl) < 0.5 ? 0 : 1;
- }
-
- double failRatio = (double)failCnt / totalCnt;
-
- System.out.println("\n>>> Fail percentage: " + (failRatio * 100) + "%.");
-
- System.out.println("\n>>> Distributed multilayer perceptron example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ // Create cache with training data.
+ CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+
+ IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
+
+ // Fill cache with training data.
+ trainingSet.put(0, new LabeledPoint(0, 0, 0));
+ trainingSet.put(1, new LabeledPoint(0, 1, 1));
+ trainingSet.put(2, new LabeledPoint(1, 0, 1));
+ trainingSet.put(3, new LabeledPoint(1, 1, 0));
+
+ // Define a layered architecture.
+ MLPArchitecture arch = new MLPArchitecture(2).
+ withAddedLayer(10, true, Activators.RELU).
+ withAddedLayer(1, false, Activators.SIGMOID);
+
+ // Define a neural network trainer.
+ MLPTrainer<SimpleGDParameterUpdate> trainer = new MLPTrainer<>(
+ arch,
+ LossFunctions.MSE,
+ new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.1),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ),
+ 3000,
+ 4,
+ 50,
+ 123L
+ );
+
+ // Train neural network and get multilayer perceptron model.
+ MultilayerPerceptron mlp = trainer.fit(
+ ignite,
+ trainingSet,
+ (k, v) -> VectorUtils.of(v.x, v.y),
+ (k, v) -> new double[] {v.lb}
+ );
+
+ int totalCnt = 4;
+ int failCnt = 0;
+
+ // Calculate score.
+ for (int i = 0; i < 4; i++) {
+ LabeledPoint pnt = trainingSet.get(i);
+ Matrix predicted = mlp.apply(new DenseMatrix(new double[][] {{pnt.x, pnt.y}}));
+
+ double predictedVal = predicted.get(0, 0);
+ double lbl = pnt.lb;
+ System.out.printf(">>> key: %d\t\t predicted: %.4f\t\tlabel: %.4f\n", i, predictedVal, lbl);
+ failCnt += Math.abs(predictedVal - lbl) < 0.5 ? 0 : 1;
+ }
+
+ double failRatio = (double)failCnt / totalCnt;
+
+ System.out.println("\n>>> Fail percentage: " + (failRatio * 100) + "%.");
+ System.out.println("\n>>> Distributed multilayer perceptron example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
index 6ac445c..862a37f 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerExample.java
@@ -29,7 +29,6 @@ import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run linear regression model based on <a href="http://web.stanford.edu/group/SOL/software/lsqr/">LSQR algorithm</a>
@@ -110,47 +109,40 @@ public class LinearRegressionLSQRTrainerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- LinearRegressionLSQRTrainerExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- System.out.println(">>> Create new linear regression trainer object.");
- LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
+ System.out.println(">>> Create new linear regression trainer object.");
+ LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
- System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ System.out.println(">>> Perform the training to get the model.");
+ LinearRegressionModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> Linear regression model: " + mdl);
+ System.out.println(">>> Linear regression model: " + mdl);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- double prediction = mdl.apply(new DenseVector(inputs));
+ double prediction = mdl.apply(new DenseVector(inputs));
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
+ }
- System.out.println(">>> ---------------------------------");
+ System.out.println(">>> ---------------------------------");
- System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
index 320d464..5692cb3 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionLSQRTrainerWithMinMaxScalerExample.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerPreprocessor
import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run linear regression model based on <a href="http://web.stanford.edu/group/SOL/software/lsqr/">LSQR algorithm</a>
@@ -116,55 +115,47 @@ public class LinearRegressionLSQRTrainerWithMinMaxScalerExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- LinearRegressionLSQRTrainerWithMinMaxScalerExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, Vector> dataCache = new TestCache(ignite).getVectors(data);
+ IgniteCache<Integer, Vector> dataCache = new TestCache(ignite).getVectors(data);
- System.out.println(">>> Create new minmaxscaling trainer object.");
- MinMaxScalerTrainer<Integer, Vector> normalizationTrainer = new MinMaxScalerTrainer<>();
+ System.out.println(">>> Create new minmaxscaling trainer object.");
+ MinMaxScalerTrainer<Integer, Vector> normalizationTrainer = new MinMaxScalerTrainer<>();
- System.out.println(">>> Perform the training to get the minmaxscaling preprocessor.");
- IgniteBiFunction<Integer, Vector, Vector> preprocessor = normalizationTrainer.fit(
- ignite,
- dataCache,
- (k, v) -> {
- double[] arr = v.asArray();
- return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
- }
- );
+ System.out.println(">>> Perform the training to get the minmaxscaling preprocessor.");
+ IgniteBiFunction<Integer, Vector, Vector> preprocessor = normalizationTrainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> {
+ double[] arr = v.asArray();
+ return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
+ }
+ );
- System.out.println(">>> Create new linear regression trainer object.");
- LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
+ System.out.println(">>> Create new linear regression trainer object.");
+ LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
- System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel mdl = trainer.fit(ignite, dataCache, preprocessor, (k, v) -> v.get(0));
+ System.out.println(">>> Perform the training to get the model.");
+ LinearRegressionModel mdl = trainer.fit(ignite, dataCache, preprocessor, (k, v) -> v.get(0));
- System.out.println(">>> Linear regression model: " + mdl);
+ System.out.println(">>> Linear regression model: " + mdl);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- try (QueryCursor<Cache.Entry<Integer, Vector>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, Vector> observation : observations) {
- Integer key = observation.getKey();
- Vector val = observation.getValue();
- double groundTruth = val.get(0);
+ try (QueryCursor<Cache.Entry<Integer, Vector>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, Vector> observation : observations) {
+ Integer key = observation.getKey();
+ Vector val = observation.getValue();
+ double groundTruth = val.get(0);
- double prediction = mdl.apply(preprocessor.apply(key, val));
+ double prediction = mdl.apply(preprocessor.apply(key, val));
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
+ }
- System.out.println(">>> ---------------------------------");
-
- System.out.println(">>> Linear regression model with minmaxscaling preprocessor over cache based dataset usage example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> Linear regression model with minmaxscaling preprocessor over cache based dataset usage example completed.");
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
index 9fdc0df..1e9bd5a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/linear/LinearRegressionSGDTrainerExample.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.optimization.updatecalculators.RPropParameterUpdate;
import org.apache.ignite.ml.optimization.updatecalculators.RPropUpdateCalculator;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.regressions.linear.LinearRegressionSGDTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run linear regression model based on based on
@@ -114,52 +113,44 @@ public class LinearRegressionSGDTrainerExample {
// Start ignite grid.
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- LinearRegressionSGDTrainerExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- System.out.println(">>> Create new linear regression trainer object.");
- LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new RPropUpdateCalculator(),
- RPropParameterUpdate::sumLocal,
- RPropParameterUpdate::avg
- ), 100000, 10, 100, 123L);
+ System.out.println(">>> Create new linear regression trainer object.");
+ LinearRegressionSGDTrainer<?> trainer = new LinearRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new RPropUpdateCalculator(),
+ RPropParameterUpdate::sumLocal,
+ RPropParameterUpdate::avg
+ ), 100000, 10, 100, 123L);
- System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ System.out.println(">>> Perform the training to get the model.");
+ LinearRegressionModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> Linear regression model: " + mdl);
+ System.out.println(">>> Linear regression model: " + mdl);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- double prediction = mdl.apply(new DenseVector(inputs));
+ double prediction = mdl.apply(new DenseVector(inputs));
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
+ }
- System.out.println(">>> ---------------------------------");
-
- System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
index 0a6ff01..8d4218d 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/binary/LogisticRegressionSGDTrainerExample.java
@@ -32,7 +32,6 @@ import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDParameterUpda
import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalculator;
import org.apache.ignite.ml.regressions.logistic.binomial.LogisticRegressionModel;
import org.apache.ignite.ml.regressions.logistic.binomial.LogisticRegressionSGDTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run logistic regression model based on <a href="https://en.wikipedia.org/wiki/Stochastic_gradient_descent">
@@ -57,69 +56,62 @@ public class LogisticRegressionSGDTrainerExample {
// Start ignite grid.
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- LogisticRegressionSGDTrainerExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- System.out.println(">>> Create new logistic regression trainer object.");
- LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ), 100000, 10, 100, 123L);
+ System.out.println(">>> Create new logistic regression trainer object.");
+ LogisticRegressionSGDTrainer<?> trainer = new LogisticRegressionSGDTrainer<>(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ), 100000, 10, 100, 123L);
- System.out.println(">>> Perform the training to get the model.");
- LogisticRegressionModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ System.out.println(">>> Perform the training to get the model.");
+ LogisticRegressionModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> Logistic regression model: " + mdl);
+ System.out.println(">>> Logistic regression model: " + mdl);
- int amountOfErrors = 0;
- int totalAmount = 0;
+ int amountOfErrors = 0;
+ int totalAmount = 0;
- // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
- int[][] confusionMtx = {{0, 0}, {0, 0}};
+ // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+ int[][] confusionMtx = {{0, 0}, {0, 0}};
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- double prediction = mdl.apply(new DenseVector(inputs));
+ double prediction = mdl.apply(new DenseVector(inputs));
- totalAmount++;
- if(groundTruth != prediction)
- amountOfErrors++;
+ totalAmount++;
+ if(groundTruth != prediction)
+ amountOfErrors++;
- int idx1 = (int)prediction;
- int idx2 = (int)groundTruth;
+ int idx1 = (int)prediction;
+ int idx2 = (int)groundTruth;
- confusionMtx[idx1][idx2]++;
+ confusionMtx[idx1][idx2]++;
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
-
- System.out.println(">>> ---------------------------------");
-
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
- System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
System.out.println(">>> ---------------------------------");
- System.out.println(">>> Logistic regression model over partitioned dataset usage example completed.");
- });
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ }
- igniteThread.start();
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+ System.out.println(">>> ---------------------------------");
- igniteThread.join();
+ System.out.println(">>> Logistic regression model over partitioned dataset usage example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/multiclass/LogRegressionMultiClassClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/multiclass/LogRegressionMultiClassClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/multiclass/LogRegressionMultiClassClassificationExample.java
index e670f01..ff2761a 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/multiclass/LogRegressionMultiClassClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/regression/logistic/multiclass/LogRegressionMultiClassClassificationExample.java
@@ -35,7 +35,6 @@ import org.apache.ignite.ml.optimization.updatecalculators.SimpleGDUpdateCalcula
import org.apache.ignite.ml.preprocessing.minmaxscaling.MinMaxScalerTrainer;
import org.apache.ignite.ml.regressions.logistic.multiclass.LogRegressionMultiClassModel;
import org.apache.ignite.ml.regressions.logistic.multiclass.LogRegressionMultiClassTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run Logistic Regression multi-class classification trainer ({@link LogRegressionMultiClassModel}) over distributed
@@ -62,115 +61,109 @@ public class LogRegressionMultiClassClassificationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- LogRegressionMultiClassClassificationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, Vector> dataCache = new TestCache(ignite).getVectors(data);
+ IgniteCache<Integer, Vector> dataCache = new TestCache(ignite).getVectors(data);
- LogRegressionMultiClassTrainer<?> trainer = new LogRegressionMultiClassTrainer<>()
- .withUpdatesStgy(new UpdatesStrategy<>(
- new SimpleGDUpdateCalculator(0.2),
- SimpleGDParameterUpdate::sumLocal,
- SimpleGDParameterUpdate::avg
- ))
- .withAmountOfIterations(100000)
- .withAmountOfLocIterations(10)
- .withBatchSize(100)
- .withSeed(123L);
+ LogRegressionMultiClassTrainer<?> trainer = new LogRegressionMultiClassTrainer<>()
+ .withUpdatesStgy(new UpdatesStrategy<>(
+ new SimpleGDUpdateCalculator(0.2),
+ SimpleGDParameterUpdate::sumLocal,
+ SimpleGDParameterUpdate::avg
+ ))
+ .withAmountOfIterations(100000)
+ .withAmountOfLocIterations(10)
+ .withBatchSize(100)
+ .withSeed(123L);
- LogRegressionMultiClassModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> {
- double[] arr = v.asArray();
- return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
- },
- (k, v) -> v.get(0)
- );
+ LogRegressionMultiClassModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> {
+ double[] arr = v.asArray();
+ return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
+ },
+ (k, v) -> v.get(0)
+ );
- System.out.println(">>> SVM Multi-class model");
- System.out.println(mdl.toString());
+ System.out.println(">>> SVM Multi-class model");
+ System.out.println(mdl.toString());
- MinMaxScalerTrainer<Integer, Vector> normalizationTrainer = new MinMaxScalerTrainer<>();
+ MinMaxScalerTrainer<Integer, Vector> normalizationTrainer = new MinMaxScalerTrainer<>();
- IgniteBiFunction<Integer, Vector, Vector> preprocessor = normalizationTrainer.fit(
- ignite,
- dataCache,
- (k, v) -> {
- double[] arr = v.asArray();
- return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
- }
- );
-
- LogRegressionMultiClassModel mdlWithNormalization = trainer.fit(
- ignite,
- dataCache,
- preprocessor,
- (k, v) -> v.get(0)
- );
+ IgniteBiFunction<Integer, Vector, Vector> preprocessor = normalizationTrainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> {
+ double[] arr = v.asArray();
+ return VectorUtils.of(Arrays.copyOfRange(arr, 1, arr.length));
+ }
+ );
- System.out.println(">>> Logistic Regression Multi-class model with minmaxscaling");
- System.out.println(mdlWithNormalization.toString());
+ LogRegressionMultiClassModel mdlWithNormalization = trainer.fit(
+ ignite,
+ dataCache,
+ preprocessor,
+ (k, v) -> v.get(0)
+ );
- System.out.println(">>> ----------------------------------------------------------------");
- System.out.println(">>> | Prediction\t| Prediction with Normalization\t| Ground Truth\t|");
- System.out.println(">>> ----------------------------------------------------------------");
+ System.out.println(">>> Logistic Regression Multi-class model with minmaxscaling");
+ System.out.println(mdlWithNormalization.toString());
- int amountOfErrors = 0;
- int amountOfErrorsWithNormalization = 0;
- int totalAmount = 0;
+ System.out.println(">>> ----------------------------------------------------------------");
+ System.out.println(">>> | Prediction\t| Prediction with Normalization\t| Ground Truth\t|");
+ System.out.println(">>> ----------------------------------------------------------------");
- // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
- int[][] confusionMtx = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
- int[][] confusionMtxWithNormalization = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
+ int amountOfErrors = 0;
+ int amountOfErrorsWithNormalization = 0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, Vector>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, Vector> observation : observations) {
- double[] val = observation.getValue().asArray();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+ int[][] confusionMtx = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
+ int[][] confusionMtxWithNormalization = {{0, 0, 0}, {0, 0, 0}, {0, 0, 0}};
- double prediction = mdl.apply(new DenseVector(inputs));
- double predictionWithNormalization = mdlWithNormalization.apply(new DenseVector(inputs));
+ try (QueryCursor<Cache.Entry<Integer, Vector>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, Vector> observation : observations) {
+ double[] val = observation.getValue().asArray();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- totalAmount++;
+ double prediction = mdl.apply(new DenseVector(inputs));
+ double predictionWithNormalization = mdlWithNormalization.apply(new DenseVector(inputs));
- // Collect data for model
- if(groundTruth != prediction)
- amountOfErrors++;
+ totalAmount++;
- int idx1 = (int)prediction == 1 ? 0 : ((int)prediction == 3 ? 1 : 2);
- int idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
+ // Collect data for model
+ if(groundTruth != prediction)
+ amountOfErrors++;
- confusionMtx[idx1][idx2]++;
+ int idx1 = (int)prediction == 1 ? 0 : ((int)prediction == 3 ? 1 : 2);
+ int idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
- // Collect data for model with minmaxscaling
- if(groundTruth != predictionWithNormalization)
- amountOfErrorsWithNormalization++;
+ confusionMtx[idx1][idx2]++;
- idx1 = (int)predictionWithNormalization == 1 ? 0 : ((int)predictionWithNormalization == 3 ? 1 : 2);
- idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
+ // Collect data for model with minmaxscaling
+ if(groundTruth != predictionWithNormalization)
+ amountOfErrorsWithNormalization++;
- confusionMtxWithNormalization[idx1][idx2]++;
+ idx1 = (int)predictionWithNormalization == 1 ? 0 : ((int)predictionWithNormalization == 3 ? 1 : 2);
+ idx2 = (int)groundTruth == 1 ? 0 : ((int)groundTruth == 3 ? 1 : 2);
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t\t\t\t\t| %.4f\t\t|\n", prediction, predictionWithNormalization, groundTruth);
- }
- System.out.println(">>> ----------------------------------------------------------------");
- System.out.println("\n>>> -----------------Logistic Regression model-------------");
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
- System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+ confusionMtxWithNormalization[idx1][idx2]++;
- System.out.println("\n>>> -----------------Logistic Regression model with Normalization-------------");
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrorsWithNormalization);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrorsWithNormalization / (double)totalAmount));
- System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtxWithNormalization));
-
- System.out.println(">>> Logistic Regression Multi-class classification model over cached dataset usage example completed.");
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t\t\t\t\t| %.4f\t\t|\n", prediction, predictionWithNormalization, groundTruth);
}
- });
+ System.out.println(">>> ----------------------------------------------------------------");
+ System.out.println("\n>>> -----------------Logistic Regression model-------------");
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+
+ System.out.println("\n>>> -----------------Logistic Regression model with Normalization-------------");
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrorsWithNormalization);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrorsWithNormalization / (double)totalAmount));
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtxWithNormalization));
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Logistic Regression Multi-class classification model over cached dataset usage example completed.");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/selection/cv/CrossValidationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/selection/cv/CrossValidationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/selection/cv/CrossValidationExample.java
index eb4c8f3..25ce156 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/selection/cv/CrossValidationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/selection/cv/CrossValidationExample.java
@@ -24,13 +24,11 @@ import org.apache.ignite.IgniteCache;
import org.apache.ignite.Ignition;
import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction;
import org.apache.ignite.configuration.CacheConfiguration;
-import org.apache.ignite.examples.ml.tree.DecisionTreeClassificationTrainerExample;
import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.selection.cv.CrossValidation;
import org.apache.ignite.ml.selection.scoring.metric.Accuracy;
import org.apache.ignite.ml.tree.DecisionTreeClassificationTrainer;
import org.apache.ignite.ml.tree.DecisionTreeNode;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run <a href="https://en.wikipedia.org/wiki/Decision_tree">decision tree</a> classification with
@@ -54,46 +52,38 @@ public class CrossValidationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- DecisionTreeClassificationTrainerExample.class.getSimpleName(), () -> {
+ // Create cache with training data.
+ CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
+ trainingSetCfg.setName("TRAINING_SET");
+ trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
- // Create cache with training data.
- CacheConfiguration<Integer, LabeledPoint> trainingSetCfg = new CacheConfiguration<>();
- trainingSetCfg.setName("TRAINING_SET");
- trainingSetCfg.setAffinity(new RendezvousAffinityFunction(false, 10));
+ IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
- IgniteCache<Integer, LabeledPoint> trainingSet = ignite.createCache(trainingSetCfg);
+ Random rnd = new Random(0);
- Random rnd = new Random(0);
+ // Fill training data.
+ for (int i = 0; i < 1000; i++)
+ trainingSet.put(i, generatePoint(rnd));
- // Fill training data.
- for (int i = 0; i < 1000; i++)
- trainingSet.put(i, generatePoint(rnd));
+ // Create classification trainer.
+ DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
- // Create classification trainer.
- DecisionTreeClassificationTrainer trainer = new DecisionTreeClassificationTrainer(4, 0);
+ CrossValidation<DecisionTreeNode, Double, Integer, LabeledPoint> scoreCalculator
+ = new CrossValidation<>();
- CrossValidation<DecisionTreeNode, Double, Integer, LabeledPoint> scoreCalculator
- = new CrossValidation<>();
+ double[] scores = scoreCalculator.score(
+ trainer,
+ new Accuracy<>(),
+ ignite,
+ trainingSet,
+ (k, v) -> VectorUtils.of(v.x, v.y),
+ (k, v) -> v.lb,
+ 4
+ );
- double[] scores = scoreCalculator.score(
- trainer,
- new Accuracy<>(),
- ignite,
- trainingSet,
- (k, v) -> VectorUtils.of(v.x, v.y),
- (k, v) -> v.lb,
- 4
- );
+ System.out.println(">>> Accuracy: " + Arrays.toString(scores));
- System.out.println(">>> Accuracy: " + Arrays.toString(scores));
-
- System.out.println(">>> Cross validation score calculator example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> Cross validation score calculator example completed.");
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
index fa1c2ca..8b104f5 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/selection/split/TrainTestDatasetSplitterExample.java
@@ -31,7 +31,6 @@ import org.apache.ignite.ml.regressions.linear.LinearRegressionLSQRTrainer;
import org.apache.ignite.ml.regressions.linear.LinearRegressionModel;
import org.apache.ignite.ml.selection.split.TrainTestDatasetSplitter;
import org.apache.ignite.ml.selection.split.TrainTestSplit;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run linear regression model over dataset split on train and test subsets ({@link TrainTestDatasetSplitter}).
@@ -113,55 +112,47 @@ public class TrainTestDatasetSplitterExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- TrainTestDatasetSplitterExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- System.out.println(">>> Create new linear regression trainer object.");
- LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
+ System.out.println(">>> Create new linear regression trainer object.");
+ LinearRegressionLSQRTrainer trainer = new LinearRegressionLSQRTrainer();
- System.out.println(">>> Create new training dataset splitter object.");
- TrainTestSplit<Integer, double[]> split = new TrainTestDatasetSplitter<Integer, double[]>()
- .split(0.75);
+ System.out.println(">>> Create new training dataset splitter object.");
+ TrainTestSplit<Integer, double[]> split = new TrainTestDatasetSplitter<Integer, double[]>()
+ .split(0.75);
- System.out.println(">>> Perform the training to get the model.");
- LinearRegressionModel mdl = trainer.fit(
- ignite,
- dataCache,
- split.getTrainFilter(),
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ System.out.println(">>> Perform the training to get the model.");
+ LinearRegressionModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ split.getTrainFilter(),
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> Linear regression model: " + mdl);
+ System.out.println(">>> Linear regression model: " + mdl);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- ScanQuery<Integer, double[]> qry = new ScanQuery<>();
- qry.setFilter(split.getTestFilter());
+ ScanQuery<Integer, double[]> qry = new ScanQuery<>();
+ qry.setFilter(split.getTestFilter());
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(qry)) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(qry)) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- double prediction = mdl.apply(new DenseVector(inputs));
+ double prediction = mdl.apply(new DenseVector(inputs));
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
+ }
- System.out.println(">>> ---------------------------------");
-
- System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
- });
-
- igniteThread.start();
-
- igniteThread.join();
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/609266fe/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
index f71db2d..c219441 100644
--- a/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/ml/svm/binary/SVMBinaryClassificationExample.java
@@ -29,7 +29,6 @@ import org.apache.ignite.ml.math.primitives.vector.VectorUtils;
import org.apache.ignite.ml.math.primitives.vector.impl.DenseVector;
import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationModel;
import org.apache.ignite.ml.svm.SVMLinearBinaryClassificationTrainer;
-import org.apache.ignite.thread.IgniteThread;
/**
* Run SVM binary-class classification model ({@link SVMLinearBinaryClassificationModel}) over distributed dataset.
@@ -54,64 +53,58 @@ public class SVMBinaryClassificationExample {
try (Ignite ignite = Ignition.start("examples/config/example-ignite.xml")) {
System.out.println(">>> Ignite grid started.");
- IgniteThread igniteThread = new IgniteThread(ignite.configuration().getIgniteInstanceName(),
- SVMBinaryClassificationExample.class.getSimpleName(), () -> {
- IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
+ IgniteCache<Integer, double[]> dataCache = new TestCache(ignite).fillCacheWith(data);
- SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer();
+ SVMLinearBinaryClassificationTrainer trainer = new SVMLinearBinaryClassificationTrainer();
- SVMLinearBinaryClassificationModel mdl = trainer.fit(
- ignite,
- dataCache,
- (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
- (k, v) -> v[0]
- );
+ SVMLinearBinaryClassificationModel mdl = trainer.fit(
+ ignite,
+ dataCache,
+ (k, v) -> VectorUtils.of(Arrays.copyOfRange(v, 1, v.length)),
+ (k, v) -> v[0]
+ );
- System.out.println(">>> SVM model " + mdl);
+ System.out.println(">>> SVM model " + mdl);
- System.out.println(">>> ---------------------------------");
- System.out.println(">>> | Prediction\t| Ground Truth\t|");
- System.out.println(">>> ---------------------------------");
-
- int amountOfErrors = 0;
- int totalAmount = 0;
+ System.out.println(">>> ---------------------------------");
+ System.out.println(">>> | Prediction\t| Ground Truth\t|");
+ System.out.println(">>> ---------------------------------");
- // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
- int[][] confusionMtx = {{0, 0}, {0, 0}};
+ int amountOfErrors = 0;
+ int totalAmount = 0;
- try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
- for (Cache.Entry<Integer, double[]> observation : observations) {
- double[] val = observation.getValue();
- double[] inputs = Arrays.copyOfRange(val, 1, val.length);
- double groundTruth = val[0];
+ // Build confusion matrix. See https://en.wikipedia.org/wiki/Confusion_matrix
+ int[][] confusionMtx = {{0, 0}, {0, 0}};
- double prediction = mdl.apply(new DenseVector(inputs));
+ try (QueryCursor<Cache.Entry<Integer, double[]>> observations = dataCache.query(new ScanQuery<>())) {
+ for (Cache.Entry<Integer, double[]> observation : observations) {
+ double[] val = observation.getValue();
+ double[] inputs = Arrays.copyOfRange(val, 1, val.length);
+ double groundTruth = val[0];
- totalAmount++;
- if(groundTruth != prediction)
- amountOfErrors++;
+ double prediction = mdl.apply(new DenseVector(inputs));
- int idx1 = prediction == 0.0 ? 0 : 1;
- int idx2 = groundTruth == 0.0 ? 0 : 1;
+ totalAmount++;
+ if(groundTruth != prediction)
+ amountOfErrors++;
- confusionMtx[idx1][idx2]++;
+ int idx1 = prediction == 0.0 ? 0 : 1;
+ int idx2 = groundTruth == 0.0 ? 0 : 1;
- System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
- }
+ confusionMtx[idx1][idx2]++;
- System.out.println(">>> ---------------------------------");
-
- System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
- System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ System.out.printf(">>> | %.4f\t\t| %.4f\t\t|\n", prediction, groundTruth);
}
- System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
+ System.out.println(">>> ---------------------------------");
+
+ System.out.println("\n>>> Absolute amount of errors " + amountOfErrors);
+ System.out.println("\n>>> Accuracy " + (1 - amountOfErrors / (double)totalAmount));
+ }
- System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
- });
+ System.out.println("\n>>> Confusion matrix is " + Arrays.deepToString(confusionMtx));
- igniteThread.start();
- igniteThread.join();
+ System.out.println(">>> Linear regression model over cache based dataset usage example completed.");
}
}
[12/21] ignite git commit: IGNITE-7764: MVCC: cache API support. This
closes #4725.
Posted by sb...@apache.org.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
index 4d5fa13..a83a93f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java
@@ -42,8 +42,6 @@ import org.apache.ignite.internal.processors.cache.GridCacheMvccCandidate;
import org.apache.ignite.internal.processors.cache.GridCacheReturn;
import org.apache.ignite.internal.processors.cache.IgniteCacheExpiryPolicy;
import org.apache.ignite.internal.processors.cache.KeyCacheObject;
-import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
-import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedCacheEntry;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedLockCancelledException;
import org.apache.ignite.internal.processors.cache.distributed.GridDistributedUnlockRequest;
@@ -61,6 +59,10 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSing
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxLocal;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearUnlockRequest;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccQueryTracker;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils;
+import org.apache.ignite.internal.processors.cache.persistence.CacheDataRow;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalEx;
import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
@@ -68,6 +70,7 @@ import org.apache.ignite.internal.util.future.GridEmbeddedFuture;
import org.apache.ignite.internal.util.future.GridFinishedFuture;
import org.apache.ignite.internal.util.lang.IgnitePair;
import org.apache.ignite.internal.util.typedef.C2;
+import org.apache.ignite.internal.util.typedef.CI1;
import org.apache.ignite.internal.util.typedef.CI2;
import org.apache.ignite.internal.util.typedef.CX1;
import org.apache.ignite.internal.util.typedef.F;
@@ -186,13 +189,14 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
if (keyCheck)
validateCacheKey(key);
- GridNearTxLocal tx = ctx.tm().threadLocalTx(ctx);
+ GridNearTxLocal tx = ctx.mvccEnabled() ? MvccUtils.tx(ctx.kernalContext()) : ctx.tm().threadLocalTx(ctx);
final CacheOperationContext opCtx = ctx.operationContextPerCall();
final boolean recovery = opCtx != null && opCtx.recovery();
- if (tx != null && !tx.implicit() && !skipTx) {
+ // Get operation bypass Tx in Mvcc mode.
+ if (!ctx.mvccEnabled() && tx != null && !tx.implicit() && !skipTx) {
return asyncOp(tx, new AsyncOp<V>() {
@Override public IgniteInternalFuture<V> op(GridNearTxLocal tx, AffinityTopologyVersion readyTopVer) {
IgniteInternalFuture<Map<Object, Object>> fut = tx.getAllAsync(ctx,
@@ -230,6 +234,26 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
subjId = ctx.subjectIdPerCall(subjId, opCtx);
+ MvccSnapshot mvccSnapshot = null;
+ MvccQueryTracker mvccTracker = null;
+
+ if (ctx.mvccEnabled()) {
+ try {
+ if (tx != null)
+ mvccSnapshot = MvccUtils.requestSnapshot(ctx, tx);
+ else {
+ mvccTracker = MvccUtils.mvccTracker(ctx, null);
+
+ mvccSnapshot = mvccTracker.snapshot();
+ }
+
+ assert mvccSnapshot != null;
+ }
+ catch (IgniteCheckedException ex) {
+ return new GridFinishedFuture<>(ex);
+ }
+ }
+
GridPartitionedSingleGetFuture fut = new GridPartitionedSingleGetFuture(ctx,
ctx.toCacheKeyObject(key),
topVer,
@@ -243,10 +267,21 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
needVer,
/*keepCacheObjects*/false,
opCtx != null && opCtx.recovery(),
- null);
+ mvccSnapshot);
fut.init();
+ if(mvccTracker != null){
+ final MvccQueryTracker mvccTracker0 = mvccTracker;
+
+ fut.listen(new CI1<IgniteInternalFuture<Object>>() {
+ @Override public void apply(IgniteInternalFuture<Object> future) {
+ if(future.isDone())
+ mvccTracker0.onDone();
+ }
+ });
+ }
+
return (IgniteInternalFuture<V>)fut;
}
@@ -270,13 +305,15 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
if (keyCheck)
validateCacheKeys(keys);
- GridNearTxLocal tx = ctx.tm().threadLocalTx(ctx);
+ GridNearTxLocal tx = (ctx.mvccEnabled()) ? MvccUtils.tx(ctx.kernalContext()) : ctx.tm().threadLocalTx(ctx);
final CacheOperationContext opCtx = ctx.operationContextPerCall();
- if (tx != null && !tx.implicit() && !skipTx) {
+ if (!ctx.mvccEnabled() && tx != null && !tx.implicit() && !skipTx) {
return asyncOp(tx, new AsyncOp<Map<K, V>>(keys) {
- @Override public IgniteInternalFuture<Map<K, V>> op(GridNearTxLocal tx, AffinityTopologyVersion readyTopVer) {
+ /** {@inheritDoc} */
+ @Override public IgniteInternalFuture<Map<K, V>> op(GridNearTxLocal tx,
+ AffinityTopologyVersion readyTopVer) {
return tx.getAllAsync(ctx,
readyTopVer,
ctx.cacheKeysView(keys),
@@ -290,14 +327,34 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
}, opCtx, /*retry*/false);
}
- AffinityTopologyVersion topVer = tx == null ? ctx.affinity().affinityTopologyVersion() : tx.topologyVersion();
-
subjId = ctx.subjectIdPerCall(subjId, opCtx);
- return loadAsync(
+ MvccSnapshot mvccSnapshot = null;
+ MvccQueryTracker mvccTracker = null;
+
+ if (ctx.mvccEnabled()) {
+ try {
+ if (tx != null)
+ mvccSnapshot = MvccUtils.requestSnapshot(ctx, tx);
+ else {
+ mvccTracker = MvccUtils.mvccTracker(ctx, null);
+
+ mvccSnapshot = mvccTracker.snapshot();
+ }
+
+ assert mvccSnapshot != null;
+ }
+ catch (IgniteCheckedException ex) {
+ return new GridFinishedFuture(ex);
+ }
+ }
+
+ AffinityTopologyVersion topVer = tx == null ? ctx.affinity().affinityTopologyVersion() : tx.topologyVersion();
+
+ IgniteInternalFuture<Map<K, V>> fut = loadAsync(
ctx.cacheKeysView(keys),
opCtx == null || !opCtx.skipStore(),
- forcePrimary,
+ forcePrimary ,
topVer,
subjId,
taskName,
@@ -305,46 +362,23 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
recovery,
skipVals ? null : expiryPolicy(opCtx != null ? opCtx.expiry() : null),
skipVals,
- needVer);
- }
-
- /**
- * @param keys Keys to load.
- * @param readThrough Read through flag.
- * @param forcePrimary Force get from primary node flag.
- * @param topVer Topology version.
- * @param subjId Subject ID.
- * @param taskName Task name.
- * @param deserializeBinary Deserialize binary flag.
- * @param expiryPlc Expiry policy.
- * @param skipVals Skip values flag.
- * @param needVer Need version.
- * @return Loaded values.
- */
- private IgniteInternalFuture<Map<K, V>> loadAsync(
- @Nullable Collection<KeyCacheObject> keys,
- boolean readThrough,
- boolean forcePrimary,
- AffinityTopologyVersion topVer,
- @Nullable UUID subjId,
- String taskName,
- boolean deserializeBinary,
- boolean recovery,
- @Nullable IgniteCacheExpiryPolicy expiryPlc,
- boolean skipVals,
- boolean needVer) {
- return loadAsync(keys,
- readThrough,
- forcePrimary,
- topVer, subjId,
- taskName,
- deserializeBinary,
- recovery,
- expiryPlc,
- skipVals,
needVer,
false,
- null);
+ mvccSnapshot);
+
+ if(mvccTracker != null){
+ final MvccQueryTracker mvccTracker0 = mvccTracker;
+
+ fut.listen(new CI1<IgniteInternalFuture<Map<K, V>>>() {
+ /** {@inheritDoc} */
+ @Override public void apply(IgniteInternalFuture<Map<K, V>> future) {
+ if(future.isDone())
+ mvccTracker0.onDone();
+ }
+ });
+ }
+
+ return fut;
}
/**
@@ -445,7 +479,9 @@ public class GridDhtColocatedCache<K, V> extends GridDhtTransactionalCacheAdapte
for (KeyCacheObject key : keys) {
if (readNoEntry) {
- CacheDataRow row = ctx.offheap().read(ctx, key);
+ CacheDataRow row = mvccSnapshot != null ?
+ ctx.offheap().mvccRead(ctx, key, mvccSnapshot) :
+ ctx.offheap().read(ctx, key);
if (row != null) {
long expireTime = row.expireTime();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
index b167f26..85a48a3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearPessimisticTxPrepareFuture.java
@@ -287,10 +287,15 @@ public class GridNearPessimisticTxPrepareFuture extends GridNearTxPrepareFutureA
boolean queryMapped = false;
- for (GridDistributedTxMapping m : F.view(tx.mappings().mappings(), CU.FILTER_QUERY_MAPPING)) {
+ assert !tx.implicitSingle() || tx.queryEnlisted(); // Non-mvcc implicit-single tx goes fast commit way.
+
+ Collection<GridDistributedTxMapping> txMappings = !tx.implicitSingle() ? tx.mappings().mappings()
+ : Collections.singleton(tx.mappings().singleMapping());
+
+ for (GridDistributedTxMapping m : F.view(txMappings, CU.FILTER_QUERY_MAPPING)) {
GridDistributedTxMapping nodeMapping = mappings.get(m.primary().id());
- if(nodeMapping == null)
+ if (nodeMapping == null)
mappings.put(m.primary().id(), m);
txMapping.addMapping(F.asList(m.primary()));
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java
index f484bd6..11f98ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxAbstractEnlistFuture.java
@@ -42,10 +42,10 @@ import org.apache.ignite.internal.processors.timeout.GridTimeoutObjectAdapter;
import org.apache.ignite.internal.transactions.IgniteTxTimeoutCheckedException;
import org.apache.ignite.internal.util.tostring.GridToStringExclude;
import org.apache.ignite.internal.util.typedef.CI1;
-import org.apache.ignite.internal.util.typedef.internal.CU;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteInClosure;
+import org.apache.ignite.lang.IgniteReducer;
import org.apache.ignite.lang.IgniteUuid;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -53,11 +53,8 @@ import org.jetbrains.annotations.Nullable;
/**
*
*/
-public abstract class GridNearTxAbstractEnlistFuture extends GridCacheCompoundIdentityFuture<Long> implements
- GridCacheVersionedFuture<Long> {
- /** */
- private static final long serialVersionUID = -6069985059301497282L;
-
+public abstract class GridNearTxAbstractEnlistFuture<T> extends GridCacheCompoundIdentityFuture<T> implements
+ GridCacheVersionedFuture<T> {
/** Done field updater. */
private static final AtomicIntegerFieldUpdater<GridNearTxAbstractEnlistFuture> DONE_UPD =
AtomicIntegerFieldUpdater.newUpdater(GridNearTxAbstractEnlistFuture.class, "done");
@@ -117,10 +114,11 @@ public abstract class GridNearTxAbstractEnlistFuture extends GridCacheCompoundId
* @param cctx Cache context.
* @param tx Transaction.
* @param timeout Timeout.
+ * @param rdc Compound future reducer.
*/
public GridNearTxAbstractEnlistFuture(
- GridCacheContext<?, ?> cctx, GridNearTxLocal tx, long timeout) {
- super(CU.longReducer());
+ GridCacheContext<?, ?> cctx, GridNearTxLocal tx, long timeout, @Nullable IgniteReducer<T, T> rdc) {
+ super(rdc);
assert cctx != null;
assert tx != null;
@@ -300,8 +298,6 @@ public abstract class GridNearTxAbstractEnlistFuture extends GridCacheCompoundId
throw new IgniteCheckedException("Future is done.");
}
-
-
/**
*/
private void mapOnTopology() {
@@ -359,7 +355,7 @@ public abstract class GridNearTxAbstractEnlistFuture extends GridCacheCompoundId
}
/** {@inheritDoc} */
- @Override protected boolean processFailure(Throwable err, IgniteInternalFuture<Long> fut) {
+ @Override protected boolean processFailure(Throwable err, IgniteInternalFuture<T> fut) {
if (ex != null || !EX_UPD.compareAndSet(this, null, err))
ex.addSuppressed(err);
@@ -367,7 +363,7 @@ public abstract class GridNearTxAbstractEnlistFuture extends GridCacheCompoundId
}
/** {@inheritDoc} */
- @Override public boolean onDone(@Nullable Long res, @Nullable Throwable err, boolean cancelled) {
+ @Override public boolean onDone(@Nullable T res, @Nullable Throwable err, boolean cancelled) {
if (!DONE_UPD.compareAndSet(this, 0, 1))
return false;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java
new file mode 100644
index 0000000..8d85bd9
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistFuture.java
@@ -0,0 +1,683 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.cluster.ClusterNode;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxAbstractEnlistFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxEnlistFuture;
+import org.apache.ignite.internal.processors.cache.distributed.dht.GridDhtTxRemote;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshotWithoutTxs;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.query.EnlistOperation;
+import org.apache.ignite.internal.processors.query.UpdateSourceIterator;
+import org.apache.ignite.internal.transactions.IgniteTxRollbackCheckedException;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.plugin.extensions.communication.Message;
+import org.jetbrains.annotations.Nullable;
+
+import static org.apache.ignite.internal.processors.cache.distributed.dht.NearTxResultHandler.createResponse;
+import static org.apache.ignite.internal.processors.cache.mvcc.MvccUtils.MVCC_OP_COUNTER_NA;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
+
+/**
+ * A future tracking requests for remote nodes transaction enlisting and locking produces by cache API operations.
+ */
+public class GridNearTxEnlistFuture extends GridNearTxAbstractEnlistFuture<GridCacheReturn> {
+ /** Default batch size. */
+ public static final int DFLT_BATCH_SIZE = 1024;
+
+ /** SkipCntr field updater. */
+ private static final AtomicIntegerFieldUpdater<GridNearTxEnlistFuture> SKIP_UPD =
+ AtomicIntegerFieldUpdater.newUpdater(GridNearTxEnlistFuture.class, "skipCntr");
+
+ /** Marker object. */
+ private static final Object FINISHED = new Object();
+
+ /** Source iterator. */
+ @GridToStringExclude
+ private final UpdateSourceIterator<?> it;
+
+ /** Batch size. */
+ private int batchSize;
+
+ /** */
+ private AtomicInteger batchCntr = new AtomicInteger();
+
+ /** */
+ @SuppressWarnings("unused")
+ @GridToStringExclude
+ private volatile int skipCntr;
+
+ /** Future result. */
+ @GridToStringExclude
+ private volatile GridCacheReturn res;
+
+ /** */
+ private final Map<UUID, Batch> batches = new ConcurrentHashMap<>();
+
+ /** Row extracted from iterator but not yet used. */
+ private Object peek;
+
+ /** Topology locked flag. */
+ private boolean topLocked;
+
+ /** Ordered batch sending flag. */
+ private final boolean sequential;
+
+ /** Filter. */
+ private final CacheEntryPredicate filter;
+
+ /** Need previous value flag. */
+ private final boolean needRes;
+
+ /**
+ * @param cctx Cache context.
+ * @param tx Transaction.
+ * @param timeout Timeout.
+ * @param it Rows iterator.
+ * @param batchSize Batch size.
+ * @param sequential Sequential locking flag.
+ * @param filter Filter.
+ * @param needRes Need previous value flag.
+ */
+ public GridNearTxEnlistFuture(GridCacheContext<?, ?> cctx,
+ GridNearTxLocal tx,
+ long timeout,
+ UpdateSourceIterator<?> it,
+ int batchSize,
+ boolean sequential,
+ @Nullable CacheEntryPredicate filter,
+ boolean needRes) {
+ super(cctx, tx, timeout, null);
+
+ this.it = it;
+ this.batchSize = batchSize > 0 ? batchSize : DFLT_BATCH_SIZE;
+ this.sequential = sequential;
+ this.filter = filter;
+ this.needRes = needRes;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void map(boolean topLocked) {
+ this.topLocked = topLocked;
+
+ sendNextBatches(null);
+ }
+
+ /**
+ * Continue iterating the data rows and form new batches.
+ *
+ * @param nodeId Node that is ready for a new batch.
+ */
+ private void sendNextBatches(@Nullable UUID nodeId) {
+ try {
+ Collection<Batch> next = continueLoop(nodeId);
+
+ if (next == null)
+ return;
+
+ boolean first = (nodeId != null);
+
+ for (Batch batch : next) {
+ ClusterNode node = batch.node();
+
+ sendBatch(node, batch, first);
+
+ if (!node.isLocal())
+ first = false;
+ }
+ }
+ catch (Throwable e) {
+ onDone(e);
+
+ if (e instanceof Error)
+ throw (Error)e;
+ }
+ }
+
+ /**
+ * Iterate data rows and form batches.
+ *
+ * @param nodeId Id of node acknowledged the last batch.
+ * @return Collection of newly completed batches.
+ * @throws IgniteCheckedException If failed.
+ */
+ private Collection<Batch> continueLoop(@Nullable UUID nodeId) throws IgniteCheckedException {
+ if (nodeId != null)
+ batches.remove(nodeId);
+
+ // Accumulate number of batches released since we got here.
+ // Let only one thread do the looping.
+ if (isDone() || SKIP_UPD.getAndIncrement(this) != 0)
+ return null;
+
+ ArrayList<Batch> res = null;
+ Batch batch = null;
+
+ boolean flush = false;
+
+ EnlistOperation op = it.operation();
+
+ while (true) {
+ while (hasNext0()) {
+ checkCompleted();
+
+ Object cur = next0();
+
+ KeyCacheObject key = cctx.toCacheKeyObject(op.isDeleteOrLock() ? cur : ((IgniteBiTuple)cur).getKey());
+
+ List<ClusterNode> nodes = cctx.affinity().nodesByKey(key, topVer);
+
+ ClusterNode node;
+
+ if (F.isEmpty(nodes) || ((node = nodes.get(0)) == null))
+ throw new ClusterTopologyCheckedException("Failed to get primary node " +
+ "[topVer=" + topVer + ", key=" + key + ']');
+
+ tx.markQueryEnlisted(null);
+
+ if (!sequential)
+ batch = batches.get(node.id());
+ else if (batch != null && !batch.node().equals(node))
+ res = markReady(res, batch);
+
+ if (batch == null)
+ batches.put(node.id(), batch = new Batch(node));
+
+ if (batch.ready()) {
+ // Can't advance further at the moment.
+ batch = null;
+
+ peek = cur;
+
+ it.beforeDetach();
+
+ flush = true;
+
+ break;
+ }
+
+ batch.add(op.isDeleteOrLock() ? key : cur,
+ op != EnlistOperation.LOCK && cctx.affinityNode() && (cctx.isReplicated() || nodes.indexOf(cctx.localNode()) > 0));
+
+ if (batch.size() == batchSize)
+ res = markReady(res, batch);
+ }
+
+ if (SKIP_UPD.decrementAndGet(this) == 0)
+ break;
+
+ skipCntr = 1;
+ }
+
+ if (flush)
+ return res;
+
+ // No data left - flush incomplete batches.
+ for (Batch batch0 : batches.values()) {
+ if (!batch0.ready()) {
+ if (res == null)
+ res = new ArrayList<>();
+
+ batch0.ready(true);
+
+ res.add(batch0);
+ }
+ }
+
+ if (batches.isEmpty())
+ onDone(this.res);
+
+ return res;
+ }
+
+ /** */
+ private Object next0() {
+ if (!hasNext0())
+ throw new NoSuchElementException();
+
+ Object cur;
+
+ if ((cur = peek) != null)
+ peek = null;
+ else
+ cur = it.next();
+
+ return cur;
+ }
+
+ /** */
+ private boolean hasNext0() {
+ if (peek == null && !it.hasNext())
+ peek = FINISHED;
+
+ return peek != FINISHED;
+ }
+
+ /**
+ * Add batch to batch collection if it is ready.
+ *
+ * @param batches Collection of batches.
+ * @param batch Batch to be added.
+ */
+ private ArrayList<Batch> markReady(ArrayList<Batch> batches, Batch batch) {
+ if (!batch.ready()) {
+ batch.ready(true);
+
+ if (batches == null)
+ batches = new ArrayList<>();
+
+ batches.add(batch);
+ }
+
+ return batches;
+ }
+
+ /**
+ * @param primaryId Primary node id.
+ * @param rows Rows.
+ * @param dhtVer Dht version assigned at primary node.
+ * @param dhtFutId Dht future id assigned at primary node.
+ */
+ private void processBatchLocalBackupKeys(UUID primaryId, List<Object> rows, GridCacheVersion dhtVer,
+ IgniteUuid dhtFutId) {
+ assert dhtVer != null;
+ assert dhtFutId != null;
+
+ EnlistOperation op = it.operation();
+
+ assert op != EnlistOperation.LOCK;
+
+ boolean keysOnly = op.isDeleteOrLock();
+
+ final ArrayList<KeyCacheObject> keys = new ArrayList<>(rows.size());
+ final ArrayList<Message> vals = keysOnly ? null : new ArrayList<>(rows.size());
+
+ for (Object row : rows) {
+ if (keysOnly)
+ keys.add(cctx.toCacheKeyObject(row));
+ else {
+ keys.add(cctx.toCacheKeyObject(((IgniteBiTuple)row).getKey()));
+ vals.add(cctx.toCacheObject(((IgniteBiTuple)row).getValue()));
+ }
+ }
+
+ try {
+ GridDhtTxRemote dhtTx = cctx.tm().tx(dhtVer);
+
+ if (dhtTx == null) {
+ dhtTx = new GridDhtTxRemote(cctx.shared(),
+ cctx.localNodeId(),
+ dhtFutId,
+ primaryId,
+ lockVer,
+ topVer,
+ dhtVer,
+ null,
+ cctx.systemTx(),
+ cctx.ioPolicy(),
+ PESSIMISTIC,
+ REPEATABLE_READ,
+ false,
+ tx.remainingTime(),
+ -1,
+ this.tx.subjectId(),
+ this.tx.taskNameHash(),
+ false);
+
+ dhtTx.mvccSnapshot(new MvccSnapshotWithoutTxs(mvccSnapshot.coordinatorVersion(),
+ mvccSnapshot.counter(), MVCC_OP_COUNTER_NA, mvccSnapshot.cleanupVersion()));
+
+ dhtTx = cctx.tm().onCreated(null, dhtTx);
+
+ if (dhtTx == null || !cctx.tm().onStarted(dhtTx)) {
+ throw new IgniteTxRollbackCheckedException("Failed to update backup " +
+ "(transaction has been completed): " + dhtVer);
+ }
+ }
+
+ dhtTx.mvccEnlistBatch(cctx, it.operation(), keys, vals, mvccSnapshot.withoutActiveTransactions());
+ }
+ catch (IgniteCheckedException e) {
+ onDone(e);
+
+ return;
+ }
+
+ sendNextBatches(primaryId);
+ }
+
+ /**
+ *
+ * @param node Node.
+ * @param batch Batch.
+ * @param first First mapping flag.
+ */
+ private void sendBatch(ClusterNode node, Batch batch, boolean first) throws IgniteCheckedException {
+ updateMappings(node);
+
+ boolean clientFirst = first && cctx.localNode().isClient() && !topLocked && !tx.hasRemoteLocks();
+
+ int batchId = batchCntr.incrementAndGet();
+
+ if (node.isLocal())
+ enlistLocal(batchId, node.id(), batch);
+ else
+ sendBatch(batchId, node.id(), batch, clientFirst);
+ }
+
+ /**
+ * Send batch request to remote data node.
+ *
+ * @param batchId Id of a batch mini-future.
+ * @param nodeId Node id.
+ * @param batchFut Mini-future for the batch.
+ * @param clientFirst {@code true} if originating node is client and it is a first request to any data node.
+ */
+ private void sendBatch(int batchId, UUID nodeId, Batch batchFut, boolean clientFirst) throws IgniteCheckedException {
+ assert batchFut != null;
+
+ GridNearTxEnlistRequest req = new GridNearTxEnlistRequest(cctx.cacheId(),
+ threadId,
+ futId,
+ batchId,
+ tx.subjectId(),
+ topVer,
+ lockVer,
+ mvccSnapshot,
+ clientFirst,
+ remainingTime(),
+ tx.remainingTime(),
+ tx.taskNameHash(),
+ batchFut.rows(),
+ it.operation(),
+ needRes,
+ filter
+ );
+
+ sendRequest(req, nodeId);
+ }
+
+ /**
+ * @param req Request.
+ * @param nodeId Remote node ID
+ * @throws IgniteCheckedException if failed to send.
+ */
+ private void sendRequest(GridCacheMessage req, UUID nodeId) throws IgniteCheckedException {
+ IgniteInternalFuture<?> txSync = cctx.tm().awaitFinishAckAsync(nodeId, tx.threadId());
+
+ if (txSync == null || txSync.isDone())
+ cctx.io().send(nodeId, req, cctx.ioPolicy());
+ else
+ txSync.listen(new CI1<IgniteInternalFuture<?>>() {
+ @Override public void apply(IgniteInternalFuture<?> future) {
+ try {
+ cctx.io().send(nodeId, req, cctx.ioPolicy());
+ }
+ catch (IgniteCheckedException e) {
+ GridNearTxEnlistFuture.this.onDone(e);
+ }
+ }
+ });
+ }
+
+ /**
+ * Enlist batch of entries to the transaction on local node.
+ *
+ * @param batchId Id of a batch mini-future.
+ * @param nodeId Node id.
+ * @param batch Batch.
+ */
+ private void enlistLocal(int batchId, UUID nodeId, Batch batch) throws IgniteCheckedException {
+ Collection<Object> rows = batch.rows();
+
+ GridDhtTxEnlistFuture fut = new GridDhtTxEnlistFuture(nodeId,
+ lockVer,
+ mvccSnapshot,
+ threadId,
+ futId,
+ batchId,
+ tx,
+ remainingTime(),
+ cctx,
+ rows,
+ it.operation(),
+ filter,
+ needRes);
+
+ updateLocalFuture(fut);
+
+ fut.listen(new CI1<IgniteInternalFuture<GridCacheReturn>>() {
+ @Override public void apply(IgniteInternalFuture<GridCacheReturn> fut) {
+ try {
+ clearLocalFuture((GridDhtTxAbstractEnlistFuture)fut);
+
+ GridNearTxEnlistResponse res = fut.error() == null ? createResponse(fut) : null;
+
+ if (checkResponse(nodeId, res, fut.error()))
+ sendNextBatches(nodeId);
+ }
+ catch (IgniteCheckedException e) {
+ checkResponse(nodeId, null, e);
+ }
+ finally {
+ CU.unwindEvicts(cctx);
+ }
+ }
+ });
+
+ fut.init();
+ }
+
+ /**
+ * @param nodeId Sender node id.
+ * @param res Response.
+ */
+ public void onResult(UUID nodeId, GridNearTxEnlistResponse res) {
+ if (checkResponse(nodeId, res, res.error())) {
+
+ Batch batch = batches.get(nodeId);
+
+ if (batch != null && !F.isEmpty(batch.localBackupRows()) && res.dhtFutureId() != null)
+ processBatchLocalBackupKeys(nodeId, batch.localBackupRows(), res.dhtVersion(), res.dhtFutureId());
+ else
+ sendNextBatches(nodeId);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean onNodeLeft(UUID nodeId) {
+ if (batches.keySet().contains(nodeId)) {
+ if (log.isDebugEnabled())
+ log.debug("Found unacknowledged batch for left node [nodeId=" + nodeId + ", fut=" +
+ this + ']');
+
+ ClusterTopologyCheckedException topEx = new ClusterTopologyCheckedException("Failed to enlist keys " +
+ "(primary node left grid, retry transaction if possible) [node=" + nodeId + ']');
+
+ topEx.retryReadyFuture(cctx.shared().nextAffinityReadyFuture(topVer));
+
+ processFailure(topEx, null);
+
+ batches.remove(nodeId);
+
+ if (batches.isEmpty()) // Wait for all pending requests.
+ onDone();
+
+ }
+
+ if (log.isDebugEnabled())
+ log.debug("Future does not have mapping for left node (ignoring) [nodeId=" + nodeId +
+ ", fut=" + this + ']');
+
+ return false;
+ }
+
+ /**
+ * @param nodeId Originating node ID.
+ * @param res Response.
+ * @param err Exception.
+ * @return {@code True} if future was completed by this call.
+ */
+ @SuppressWarnings("unchecked")
+ public boolean checkResponse(UUID nodeId, GridNearTxEnlistResponse res, Throwable err) {
+ assert res != null || err != null : this;
+
+ if (err == null && res.error() != null)
+ err = res.error();
+
+ if (X.hasCause(err, ClusterTopologyCheckedException.class))
+ tx.removeMapping(nodeId);
+
+ if (err != null)
+ processFailure(err, null);
+
+ if (ex != null) {
+ batches.remove(nodeId);
+
+ if (batches.isEmpty()) // Wait for all pending requests.
+ onDone();
+
+ return false;
+ }
+
+ assert res != null;
+
+ this.res = res.result();
+
+ assert this.res != null && (this.res.emptyResult() || needRes || !this.res.success());
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GridNearTxEnlistFuture.class, this, super.toString());
+ }
+
+ /**
+ * A batch of rows
+ */
+ private static class Batch {
+ /** Node ID. */
+ @GridToStringExclude
+ private final ClusterNode node;
+
+ /** Rows. */
+ private List<Object> rows = new ArrayList<>();
+
+ /** Local backup rows. */
+ private List<Object> locBkpRows;
+
+ /** Readiness flag. Set when batch is full or no new rows are expected. */
+ private boolean ready;
+
+ /**
+ * @param node Cluster node.
+ */
+ private Batch(ClusterNode node) {
+ this.node = node;
+ }
+
+ /**
+ * @return Node.
+ */
+ public ClusterNode node() {
+ return node;
+ }
+
+ /**
+ * Adds a row.
+ *
+ * @param row Row.
+ * @param localBackup {@code true}, when the row key has local backup.
+ */
+ public void add(Object row, boolean localBackup) {
+ rows.add(row);
+
+ if (localBackup) {
+ if (locBkpRows == null)
+ locBkpRows = new ArrayList<>();
+
+ locBkpRows.add(row);
+ }
+ }
+
+ /**
+ * @return number of rows.
+ */
+ public int size() {
+ return rows.size();
+ }
+
+ /**
+ * @return Collection of rows.
+ */
+ public Collection<Object> rows() {
+ return rows;
+ }
+
+ /**
+ * @return Collection of local backup rows.
+ */
+ public List<Object> localBackupRows() {
+ return locBkpRows;
+ }
+
+ /**
+ * @return Readiness flag.
+ */
+ public boolean ready() {
+ return ready;
+ }
+
+ /**
+ * Sets readiness flag.
+ *
+ * @param ready Flag value.
+ */
+ public void ready(boolean ready) {
+ this.ready = ready;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java
new file mode 100644
index 0000000..1d87023
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistRequest.java
@@ -0,0 +1,642 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.CacheObject;
+import org.apache.ignite.internal.processors.cache.CacheObjectContext;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheIdMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.query.EnlistOperation;
+import org.apache.ignite.internal.util.tostring.GridToStringExclude;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Request to enlist into transaction and acquire locks for entries produced with Cache API operations.
+ *
+ * One request per batch of entries is used.
+ */
+public class GridNearTxEnlistRequest extends GridCacheIdMessage {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** */
+ private long threadId;
+
+ /** Future id. */
+ private IgniteUuid futId;
+
+ /** */
+ private boolean clientFirst;
+
+ /** */
+ private int miniId;
+
+ /** */
+ private UUID subjId;
+
+ /** */
+ private AffinityTopologyVersion topVer;
+
+ /** */
+ private GridCacheVersion lockVer;
+
+ /** Mvcc snapshot. */
+ private MvccSnapshot mvccSnapshot;
+
+ /** */
+ private long timeout;
+
+ /** */
+ private long txTimeout;
+
+ /** */
+ private int taskNameHash;
+
+ /** Rows to enlist. */
+ @GridDirectTransient
+ private Collection<Object> rows;
+
+ /** Serialized rows keys. */
+ @GridToStringExclude
+ private KeyCacheObject[] keys;
+
+ /** Serialized rows values. */
+ @GridToStringExclude
+ private CacheObject[] values;
+
+ /** Enlist operation. */
+ private EnlistOperation op;
+
+ /** Filter. */
+ @GridToStringExclude
+ private CacheEntryPredicate filter;
+
+ /** Need previous value flag. */
+ private boolean needRes;
+
+ /**
+ * Default constructor.
+ */
+ public GridNearTxEnlistRequest() {
+ // No-op.
+ }
+
+ /**
+ * Constructor.
+ *
+ * @param cacheId Cache id.
+ * @param threadId Thread id.
+ * @param futId Future id.
+ * @param miniId Mini-future id.
+ * @param subjId Transaction subject id.
+ * @param topVer Topology version.
+ * @param lockVer Lock version.
+ * @param mvccSnapshot Mvcc snapshot.
+ * @param clientFirst First client request flag.
+ * @param timeout Timeout.
+ * @param txTimeout Tx timeout.
+ * @param taskNameHash Task name hash.
+ * @param rows Rows.
+ * @param op Operation.
+ * @param filter Filter.
+ */
+ GridNearTxEnlistRequest(int cacheId,
+ long threadId,
+ IgniteUuid futId,
+ int miniId,
+ UUID subjId,
+ AffinityTopologyVersion topVer,
+ GridCacheVersion lockVer,
+ MvccSnapshot mvccSnapshot,
+ boolean clientFirst,
+ long timeout,
+ long txTimeout,
+ int taskNameHash,
+ Collection<Object> rows,
+ EnlistOperation op,
+ boolean needRes,
+ @Nullable CacheEntryPredicate filter) {
+ this.txTimeout = txTimeout;
+ this.filter = filter;
+ this.cacheId = cacheId;
+ this.threadId = threadId;
+ this.futId = futId;
+ this.miniId = miniId;
+ this.subjId = subjId;
+ this.topVer = topVer;
+ this.lockVer = lockVer;
+ this.mvccSnapshot = mvccSnapshot;
+ this.clientFirst = clientFirst;
+ this.timeout = timeout;
+ this.taskNameHash = taskNameHash;
+ this.rows = rows;
+ this.op = op;
+ this.needRes = needRes;
+ }
+
+ /**
+ * @return Thread id.
+ */
+ public long threadId() {
+ return threadId;
+ }
+
+ /**
+ * @return Future id.
+ */
+ public IgniteUuid futureId() {
+ return futId;
+ }
+
+ /**
+ * @return Mini future ID.
+ */
+ public int miniId() {
+ return miniId;
+ }
+
+ /**
+ * @return Subject id.
+ */
+ public UUID subjectId() {
+ return subjId;
+ }
+
+ /**
+ * @return Topology version.
+ */
+ @Override public AffinityTopologyVersion topologyVersion() {
+ return topVer;
+ }
+
+ /**
+ * @return Lock version.
+ */
+ public GridCacheVersion version() {
+ return lockVer;
+ }
+
+ /**
+ * @return MVCC snapshot.
+ */
+ public MvccSnapshot mvccSnapshot() {
+ return mvccSnapshot;
+ }
+
+ /**
+ * @return Timeout milliseconds.
+ */
+ public long timeout() {
+ return timeout;
+ }
+
+ /**
+ * @return Tx timeout milliseconds.
+ */
+ public long txTimeout() {
+ return txTimeout;
+ }
+
+ /**
+ * @return Task name hash.
+ */
+ public int taskNameHash() {
+ return taskNameHash;
+ }
+
+ /**
+ * @return {@code True} if this is the first client request.
+ */
+ public boolean firstClientRequest() {
+ return clientFirst;
+ }
+
+ /**
+ * @return Collection of rows.
+ */
+ public Collection<Object> rows() {
+ return rows;
+ }
+
+ /**
+ * @return Operation.
+ */
+ public EnlistOperation operation() {
+ return op;
+ }
+
+ /**
+ * @return Need result flag.
+ */
+ public boolean needRes() {
+ return needRes;
+ }
+
+ /**
+ * @return Filter.
+ */
+ public CacheEntryPredicate filter() {
+ return filter;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
+ super.prepareMarshal(ctx);
+
+ GridCacheContext cctx = ctx.cacheContext(cacheId);
+ CacheObjectContext objCtx = cctx.cacheObjectContext();
+
+ if (rows != null && keys == null) {
+ keys = new KeyCacheObject[rows.size()];
+
+ int i = 0;
+
+ boolean keysOnly = op.isDeleteOrLock();
+
+ values = keysOnly ? null : new CacheObject[keys.length];
+
+ for (Object row : rows) {
+ Object key, val = null;
+
+ if (keysOnly)
+ key = row;
+ else {
+ key = ((IgniteBiTuple)row).getKey();
+ val = ((IgniteBiTuple)row).getValue();
+ }
+
+ assert key != null && (keysOnly || val != null) : "key=" + key + ", val=" + val;
+
+ KeyCacheObject key0 = cctx.toCacheKeyObject(key);
+
+ assert key0 != null;
+
+ key0.prepareMarshal(objCtx);
+
+ keys[i] = key0;
+
+ if (!keysOnly) {
+ CacheObject val0 = cctx.toCacheObject(val);
+
+ assert val0 != null;
+
+ val0.prepareMarshal(objCtx);
+
+ values[i] = val0;
+ }
+
+ i++;
+ }
+ }
+
+ if (filter != null)
+ filter.prepareMarshal(cctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
+ super.finishUnmarshal(ctx, ldr);
+
+ if (keys != null) {
+ rows = new ArrayList<>(keys.length);
+
+ CacheObjectContext objCtx = ctx.cacheContext(cacheId).cacheObjectContext();
+
+ for (int i = 0; i < keys.length; i++) {
+ keys[i].finishUnmarshal(objCtx, ldr);
+
+ if (op.isDeleteOrLock())
+ rows.add(keys[i]);
+ else {
+ if (values[i] != null)
+ values[i].finishUnmarshal(objCtx, ldr);
+
+ rows.add(new IgniteBiTuple<>(keys[i], values[i]));
+ }
+ }
+
+ keys = null;
+ values = null;
+ }
+
+ if (filter != null)
+ filter.finishUnmarshal(ctx.cacheContext(cacheId), ldr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 3:
+ if (!writer.writeBoolean("clientFirst", clientFirst))
+ return false;
+
+ writer.incrementState();
+
+ case 4:
+ if (!writer.writeMessage("filter", filter))
+ return false;
+
+ writer.incrementState();
+
+ case 5:
+ if (!writer.writeIgniteUuid("futId", futId))
+ return false;
+
+ writer.incrementState();
+
+ case 6:
+ if (!writer.writeObjectArray("keys", keys, MessageCollectionItemType.MSG))
+ return false;
+
+ writer.incrementState();
+
+ case 7:
+ if (!writer.writeMessage("lockVer", lockVer))
+ return false;
+
+ writer.incrementState();
+
+ case 8:
+ if (!writer.writeInt("miniId", miniId))
+ return false;
+
+ writer.incrementState();
+
+ case 9:
+ if (!writer.writeMessage("mvccSnapshot", mvccSnapshot))
+ return false;
+
+ writer.incrementState();
+
+ case 10:
+ if (!writer.writeBoolean("needRes", needRes))
+ return false;
+
+ writer.incrementState();
+
+ case 11:
+ if (!writer.writeByte("op", op != null ? (byte)op.ordinal() : -1))
+ return false;
+
+ writer.incrementState();
+
+ case 12:
+ if (!writer.writeUuid("subjId", subjId))
+ return false;
+
+ writer.incrementState();
+
+ case 13:
+ if (!writer.writeInt("taskNameHash", taskNameHash))
+ return false;
+
+ writer.incrementState();
+
+ case 14:
+ if (!writer.writeLong("threadId", threadId))
+ return false;
+
+ writer.incrementState();
+
+ case 15:
+ if (!writer.writeLong("timeout", timeout))
+ return false;
+
+ writer.incrementState();
+
+ case 16:
+ if (!writer.writeMessage("topVer", topVer))
+ return false;
+
+ writer.incrementState();
+
+ case 17:
+ if (!writer.writeLong("txTimeout", txTimeout))
+ return false;
+
+ writer.incrementState();
+
+ case 18:
+ if (!writer.writeObjectArray("values", values, MessageCollectionItemType.MSG))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ switch (reader.state()) {
+ case 3:
+ clientFirst = reader.readBoolean("clientFirst");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 4:
+ filter = reader.readMessage("filter");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 5:
+ futId = reader.readIgniteUuid("futId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 6:
+ keys = reader.readObjectArray("keys", MessageCollectionItemType.MSG, KeyCacheObject.class);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 7:
+ lockVer = reader.readMessage("lockVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 8:
+ miniId = reader.readInt("miniId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 9:
+ mvccSnapshot = reader.readMessage("mvccSnapshot");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 10:
+ needRes = reader.readBoolean("needRes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 11:
+ byte opOrd;
+
+ opOrd = reader.readByte("op");
+
+ if (!reader.isLastRead())
+ return false;
+
+ op = EnlistOperation.fromOrdinal(opOrd);
+
+ reader.incrementState();
+
+ case 12:
+ subjId = reader.readUuid("subjId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 13:
+ taskNameHash = reader.readInt("taskNameHash");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 14:
+ threadId = reader.readLong("threadId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 15:
+ timeout = reader.readLong("timeout");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 16:
+ topVer = reader.readMessage("topVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 17:
+ txTimeout = reader.readLong("txTimeout");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 18:
+ values = reader.readObjectArray("values", MessageCollectionItemType.MSG, CacheObject.class);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(GridNearTxEnlistRequest.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 19;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean addDeploymentInfo() {
+ return false;
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 159;
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GridNearTxEnlistRequest.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java
new file mode 100644
index 0000000..4f4bbb6
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTxEnlistResponse.java
@@ -0,0 +1,372 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.near;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.GridDirectCollection;
+import org.apache.ignite.internal.GridDirectTransient;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheIdMessage;
+import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.GridCacheSharedContext;
+import org.apache.ignite.internal.processors.cache.distributed.dht.ExceptionAware;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.plugin.extensions.communication.MessageCollectionItemType;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.MessageWriter;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * A response to {@link GridNearTxEnlistRequest}.
+ */
+public class GridNearTxEnlistResponse extends GridCacheIdMessage implements ExceptionAware {
+ /** */
+ private static final long serialVersionUID = 0L;
+
+ /** Future ID. */
+ private IgniteUuid futId;
+
+ /** Error. */
+ @GridDirectTransient
+ private Throwable err;
+
+ /** Serialized error. */
+ private byte[] errBytes;
+
+ /** Mini future id. */
+ private int miniId;
+
+ /** Result. */
+ private GridCacheReturn res;
+
+ /** */
+ private GridCacheVersion lockVer;
+
+ /** */
+ private GridCacheVersion dhtVer;
+
+ /** */
+ private IgniteUuid dhtFutId;
+
+ /** New DHT nodes involved into transaction. */
+ @GridDirectCollection(UUID.class)
+ private Collection<UUID> newDhtNodes;
+
+ /**
+ * Default constructor.
+ */
+ public GridNearTxEnlistResponse() {
+ // No-op.
+ }
+
+ /**
+ * Constructor for normal result.
+ *
+ * @param cacheId Cache id.
+ * @param futId Future id.
+ * @param miniId Mini future id.
+ * @param lockVer Lock version.
+ * @param res Result.
+ * @param dhtVer Dht version.
+ * @param dhtFutId Dht future id.
+ * @param newDhtNodes New DHT nodes involved into transaction.
+ */
+ public GridNearTxEnlistResponse(int cacheId,
+ IgniteUuid futId,
+ int miniId,
+ GridCacheVersion lockVer,
+ GridCacheReturn res,
+ GridCacheVersion dhtVer,
+ IgniteUuid dhtFutId,
+ Set<UUID> newDhtNodes) {
+ this.cacheId = cacheId;
+ this.futId = futId;
+ this.miniId = miniId;
+ this.lockVer = lockVer;
+ this.res = res;
+ this.dhtVer = dhtVer;
+ this.dhtFutId = dhtFutId;
+ this.newDhtNodes = newDhtNodes;
+ }
+
+ /**
+ * Constructor for error result.
+ *
+ * @param cacheId Cache id.
+ * @param futId Future id.
+ * @param miniId Mini future id.
+ * @param lockVer Lock version.
+ * @param err Error.
+ */
+ public GridNearTxEnlistResponse(int cacheId, IgniteUuid futId, int miniId, GridCacheVersion lockVer,
+ Throwable err) {
+ this.cacheId = cacheId;
+ this.futId = futId;
+ this.miniId = miniId;
+ this.lockVer = lockVer;
+ this.err = err;
+ }
+
+ /**
+ * @return Loc version.
+ */
+ public GridCacheVersion version() {
+ return lockVer;
+ }
+
+ /**
+ * @return Future id.
+ */
+ public IgniteUuid futureId() {
+ return futId;
+ }
+
+ /**
+ * @return Mini future id.
+ */
+ public int miniId() {
+ return miniId;
+ }
+
+ /**
+ * @return Result.
+ */
+ public GridCacheReturn result() {
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Throwable error() {
+ return err;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean addDeploymentInfo() {
+ return false;
+ }
+
+ /**
+ * @return Dht version.
+ */
+ public GridCacheVersion dhtVersion() {
+ return dhtVer;
+ }
+
+ /**
+ * @return Dht future id.
+ */
+ public IgniteUuid dhtFutureId() {
+ return dhtFutId;
+ }
+
+ /** {@inheritDoc} */
+ @Override public byte fieldsCount() {
+ return 11;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean writeTo(ByteBuffer buf, MessageWriter writer) {
+ writer.setBuffer(buf);
+
+ if (!super.writeTo(buf, writer))
+ return false;
+
+ if (!writer.isHeaderWritten()) {
+ if (!writer.writeHeader(directType(), fieldsCount()))
+ return false;
+
+ writer.onHeaderWritten();
+ }
+
+ switch (writer.state()) {
+ case 3:
+ if (!writer.writeIgniteUuid("dhtFutId", dhtFutId))
+ return false;
+
+ writer.incrementState();
+
+ case 4:
+ if (!writer.writeMessage("dhtVer", dhtVer))
+ return false;
+
+ writer.incrementState();
+
+ case 5:
+ if (!writer.writeByteArray("errBytes", errBytes))
+ return false;
+
+ writer.incrementState();
+
+ case 6:
+ if (!writer.writeIgniteUuid("futId", futId))
+ return false;
+
+ writer.incrementState();
+
+ case 7:
+ if (!writer.writeMessage("lockVer", lockVer))
+ return false;
+
+ writer.incrementState();
+
+ case 8:
+ if (!writer.writeInt("miniId", miniId))
+ return false;
+
+ writer.incrementState();
+
+ case 9:
+ if (!writer.writeCollection("newDhtNodes", newDhtNodes, MessageCollectionItemType.UUID))
+ return false;
+
+ writer.incrementState();
+
+ case 10:
+ if (!writer.writeMessage("res", res))
+ return false;
+
+ writer.incrementState();
+
+ }
+
+ return true;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean readFrom(ByteBuffer buf, MessageReader reader) {
+ reader.setBuffer(buf);
+
+ if (!reader.beforeMessageRead())
+ return false;
+
+ if (!super.readFrom(buf, reader))
+ return false;
+
+ switch (reader.state()) {
+ case 3:
+ dhtFutId = reader.readIgniteUuid("dhtFutId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 4:
+ dhtVer = reader.readMessage("dhtVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 5:
+ errBytes = reader.readByteArray("errBytes");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 6:
+ futId = reader.readIgniteUuid("futId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 7:
+ lockVer = reader.readMessage("lockVer");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 8:
+ miniId = reader.readInt("miniId");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 9:
+ newDhtNodes = reader.readCollection("newDhtNodes", MessageCollectionItemType.UUID);
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ case 10:
+ res = reader.readMessage("res");
+
+ if (!reader.isLastRead())
+ return false;
+
+ reader.incrementState();
+
+ }
+
+ return reader.afterMessageRead(GridNearTxEnlistResponse.class);
+ }
+
+ /** {@inheritDoc} */
+ @Override public short directType() {
+ return 160;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void prepareMarshal(GridCacheSharedContext ctx) throws IgniteCheckedException {
+ super.prepareMarshal(ctx);
+
+ GridCacheContext cctx = ctx.cacheContext(cacheId);
+
+ if (err != null && errBytes == null)
+ errBytes = U.marshal(ctx.marshaller(), err);
+
+ if (res != null)
+ res.prepareMarshal(cctx);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr) throws IgniteCheckedException {
+ super.finishUnmarshal(ctx, ldr);
+
+ GridCacheContext cctx = ctx.cacheContext(cacheId);
+
+ if (errBytes != null)
+ err = U.unmarshal(ctx, errBytes, U.resolveClassLoader(ldr, ctx.gridConfig()));
+
+ if (res != null)
+ res.finishUnmarshal(cctx, ldr);
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GridNearTxEnlistResponse.class, this);
+ }
+}
[07/21] ignite git commit: IGNITE-9501 Backward compatibility fix -
Fixes #4860.
Posted by sb...@apache.org.
IGNITE-9501 Backward compatibility fix - Fixes #4860.
Signed-off-by: Dmitriy Govorukhin <dm...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d5f6e50e
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d5f6e50e
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d5f6e50e
Branch: refs/heads/ignite-gg-14206
Commit: d5f6e50edaa1dfd27d9a6f2a3bbcdacaf26057cd
Parents: 609266f
Author: Pavel Kovalenko <jo...@gmail.com>
Authored: Fri Sep 28 12:06:52 2018 +0300
Committer: Dmitriy Govorukhin <dm...@gmail.com>
Committed: Fri Sep 28 12:06:52 2018 +0300
----------------------------------------------------------------------
.../dht/preloader/GridDhtPartitionsExchangeFuture.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d5f6e50e/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
index 4e76f99..f43afa0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/preloader/GridDhtPartitionsExchangeFuture.java
@@ -1508,7 +1508,7 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
releaseLatch.countDown();
// For compatibility with old version where joining nodes are not waiting for latch.
- if (!cctx.exchange().latch().canSkipJoiningNodes(initialVersion()))
+ if (localJoinExchange() && !cctx.exchange().latch().canSkipJoiningNodes(initialVersion()))
return;
try {
@@ -2543,7 +2543,6 @@ public class GridDhtPartitionsExchangeFuture extends GridDhtTopologyFutureAdapte
}
}
}
-
if (allReceived) {
if (!awaitSingleMapUpdates())
return;
[13/21] ignite git commit: IGNITE-7764: MVCC: cache API support. This
closes #4725.
Posted by sb...@apache.org.
IGNITE-7764: MVCC: cache API support. This closes #4725.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/f7f834bf
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/f7f834bf
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/f7f834bf
Branch: refs/heads/ignite-gg-14206
Commit: f7f834bfaf8c4170ab852e829554c8ab5b373b77
Parents: 6f39115
Author: AMRepo <an...@gmail.com>
Authored: Fri Sep 28 15:57:24 2018 +0300
Committer: devozerov <vo...@gridgain.com>
Committed: Fri Sep 28 15:57:24 2018 +0300
----------------------------------------------------------------------
.../ignite/codegen/MessageCodeGenerator.java | 6 +-
.../communication/GridIoMessageFactory.java | 14 +-
.../processors/cache/GridCacheAdapter.java | 29 +-
.../processors/cache/GridCacheEntryEx.java | 12 +-
.../processors/cache/GridCacheMapEntry.java | 164 +++--
.../cache/GridCacheUpdateTxResult.java | 23 +-
.../cache/IgniteCacheOffheapManager.java | 24 +-
.../cache/IgniteCacheOffheapManagerImpl.java | 48 +-
.../processors/cache/IgniteCacheProxyImpl.java | 2 +-
.../dht/GridDhtTransactionalCacheAdapter.java | 134 +++-
.../dht/GridDhtTxAbstractEnlistFuture.java | 72 +-
.../distributed/dht/GridDhtTxEnlistFuture.java | 147 ++++
.../dht/GridDhtTxQueryAbstractEnlistFuture.java | 83 +++
.../dht/GridDhtTxQueryEnlistFuture.java | 23 +-
.../dht/GridDhtTxQueryResultsEnlistFuture.java | 37 +-
.../cache/distributed/dht/GridDhtTxRemote.java | 4 +
.../dht/GridPartitionedSingleGetFuture.java | 3 +-
.../distributed/dht/NearTxResultHandler.java | 128 ++++
.../dht/colocated/GridDhtColocatedCache.java | 138 ++--
.../GridNearPessimisticTxPrepareFuture.java | 9 +-
.../near/GridNearTxAbstractEnlistFuture.java | 20 +-
.../near/GridNearTxEnlistFuture.java | 683 +++++++++++++++++++
.../near/GridNearTxEnlistRequest.java | 642 +++++++++++++++++
.../near/GridNearTxEnlistResponse.java | 372 ++++++++++
.../cache/distributed/near/GridNearTxLocal.java | 416 +++++++++--
.../GridNearTxQueryAbstractEnlistFuture.java | 36 +
.../near/GridNearTxQueryEnlistFuture.java | 4 +-
.../near/GridNearTxQueryEnlistResponse.java | 3 +-
.../GridNearTxQueryResultsEnlistFuture.java | 5 +-
.../GridNearTxQueryResultsEnlistResponse.java | 2 +-
.../cache/mvcc/MvccProcessorImpl.java | 3 +-
.../cache/mvcc/MvccQueryTrackerImpl.java | 10 +-
.../persistence/GridCacheOffheapManager.java | 15 +-
.../transactions/IgniteTxLocalAdapter.java | 5 +-
.../cache/transactions/IgniteTxManager.java | 6 +-
.../cache/tree/mvcc/data/MvccUpdateDataRow.java | 88 ++-
.../cache/tree/mvcc/data/ResultType.java | 4 +-
.../processors/cache/GridCacheTestEntryEx.java | 7 +-
.../IgniteCacheTxIteratorSelfTest.java | 10 +
...vccAbstractBasicCoordinatorFailoverTest.java | 25 +-
...acheMvccAbstractCoordinatorFailoverTest.java | 21 -
.../mvcc/CacheMvccAbstractFeatureTest.java | 2 +-
.../cache/mvcc/CacheMvccAbstractTest.java | 132 ++--
.../cache/mvcc/CacheMvccTransactionsTest.java | 596 +++++++---------
.../DataStreamProcessorMvccSelfTest.java | 5 +
.../configvariations/ConfigVariations.java | 1 -
.../query/h2/DhtResultSetEnlistFuture.java | 4 +-
.../query/h2/NearResultSetEnlistFuture.java | 3 -
...sactionsCommandsWithMvccEnabledSelfTest.java | 78 +--
...cheMvccSelectForUpdateQueryAbstractTest.java | 2 +
.../mvcc/CacheMvccSqlQueriesAbstractTest.java | 4 +
.../mvcc/CacheMvccSqlTxQueriesAbstractTest.java | 15 +-
.../mvcc/MvccRepeatableReadBulkOpsTest.java | 441 ++++++++++++
.../mvcc/MvccRepeatableReadOperationsTest.java | 276 ++++++++
.../testsuites/IgniteCacheMvccSqlTestSuite.java | 6 +
.../ApiParity/IgniteConfigurationParityTest.cs | 5 +-
56 files changed, 4168 insertions(+), 879 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
----------------------------------------------------------------------
diff --git a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
index 9512bae..bcb9ef4 100644
--- a/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
+++ b/modules/codegen/src/main/java/org/apache/ignite/codegen/MessageCodeGenerator.java
@@ -38,12 +38,14 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.UUID;
-import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.GridCodegenConverter;
import org.apache.ignite.internal.GridDirectCollection;
import org.apache.ignite.internal.GridDirectMap;
import org.apache.ignite.internal.GridDirectTransient;
import org.apache.ignite.internal.IgniteCodeGeneratingFail;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistResponse;
+import org.apache.ignite.internal.util.IgniteUtils;
import org.apache.ignite.internal.util.typedef.internal.SB;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteUuid;
@@ -168,7 +170,7 @@ public class MessageCodeGenerator {
// gen.generateAll(true);
-// gen.generateAndWrite(GridNearTxQueryResultsEnlistRequest.class);
+ gen.generateAndWrite(GridNearTxEnlistResponse.class);
// gen.generateAndWrite(GridNearAtomicUpdateRequest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
index 41c75be..389d8c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/GridIoMessageFactory.java
@@ -110,6 +110,8 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLock
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetResponse;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxFinishResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest;
@@ -1066,7 +1068,17 @@ public class GridIoMessageFactory implements MessageFactory {
break;
- // [-3..119] [124..129] [-23..-27] [-36..-55]- this
+ case 159:
+ msg = new GridNearTxEnlistRequest();
+
+ break;
+
+ case 160:
+ msg = new GridNearTxEnlistResponse();
+
+ break;
+
+ // [-3..119] [124..129] [-23..-27] [-36..-55]- this
// [120..123] - DR
// [-4..-22, -30..-35] - SQL
// [2048..2053] - Snapshots
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
index 476b083..cf9337b 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java
@@ -154,7 +154,9 @@ import static org.apache.ignite.internal.processors.dr.GridDrType.DR_NONE;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_NO_FAILOVER;
import static org.apache.ignite.internal.processors.task.GridTaskThreadContextKey.TC_SUBGRID;
import static org.apache.ignite.transactions.TransactionConcurrency.OPTIMISTIC;
+import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC;
import static org.apache.ignite.transactions.TransactionIsolation.READ_COMMITTED;
+import static org.apache.ignite.transactions.TransactionIsolation.REPEATABLE_READ;
/**
* Adapter for different cache implementations.
@@ -1940,6 +1942,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
}
if (tx == null || tx.implicit()) {
+ assert !ctx.mvccEnabled() || mvccSnapshot != null;
+
Map<KeyCacheObject, EntryGetResult> misses = null;
Set<GridCacheEntryEx> newLocalEntries = null;
@@ -1978,7 +1982,8 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
boolean skipEntry = readNoEntry;
if (readNoEntry) {
- CacheDataRow row = mvccSnapshot != null ? ctx.offheap().mvccRead(ctx, key, mvccSnapshot) :
+ CacheDataRow row = mvccSnapshot != null ?
+ ctx.offheap().mvccRead(ctx, key, mvccSnapshot) :
ctx.offheap().read(ctx, key);
if (row != null) {
@@ -3411,7 +3416,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKeys(keys);
- //TODO IGNITE-7764
+ //TODO: IGNITE-9324: add explicit locks support.
MvccUtils.verifyMvccOperationSupport(ctx, "Lock");
IgniteInternalFuture<Boolean> fut = lockAllAsync(keys, timeout);
@@ -3442,7 +3447,7 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
if (keyCheck)
validateCacheKey(key);
- //TODO IGNITE-7764
+ //TODO: IGNITE-9324: add explicit locks support.
MvccUtils.verifyMvccOperationSupport(ctx, "Lock");
return lockAllAsync(Collections.singletonList(key), timeout);
@@ -4213,11 +4218,11 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
true,
op.single(),
ctx.systemTx() ? ctx : null,
- OPTIMISTIC,
- READ_COMMITTED,
+ ctx.mvccEnabled() ? PESSIMISTIC : OPTIMISTIC,
+ ctx.mvccEnabled() ? REPEATABLE_READ : READ_COMMITTED,
tCfg.getDefaultTxTimeout(),
!ctx.skipStore(),
- false,
+ ctx.mvccEnabled(),
0,
null
);
@@ -4315,11 +4320,11 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
true,
op.single(),
ctx.systemTx() ? ctx : null,
- OPTIMISTIC,
- READ_COMMITTED,
+ ctx.mvccEnabled() ? PESSIMISTIC : OPTIMISTIC,
+ ctx.mvccEnabled() ? REPEATABLE_READ : READ_COMMITTED,
txCfg.getDefaultTxTimeout(),
!skipStore,
- false,
+ ctx.mvccEnabled(),
0,
null);
@@ -4996,11 +5001,11 @@ public abstract class GridCacheAdapter<K, V> implements IgniteInternalCache<K, V
true,
op.single(),
ctx.systemTx() ? ctx : null,
- OPTIMISTIC,
- READ_COMMITTED,
+ ctx.mvccEnabled() ? PESSIMISTIC : OPTIMISTIC,
+ ctx.mvccEnabled() ? REPEATABLE_READ : READ_COMMITTED,
CU.transactionConfiguration(ctx, ctx.kernalContext().config()).getDefaultTxTimeout(),
opCtx == null || !opCtx.skipStore(),
- false,
+ ctx.mvccEnabled(),
0,
null);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
index 18fa820..2e96a9c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheEntryEx.java
@@ -352,6 +352,8 @@ public interface GridCacheEntryEx {
* @param op Cache operation.
* @param needHistory Whether to collect rows created or affected by the current tx.
* @param noCreate Entry should not be created when enabled, e.g. SQL INSERT.
+ * @param filter Filter.
+ * @param retVal Previous value return flag.
* @return Tuple containing success flag and old value. If success is {@code false},
* then value is {@code null}.
* @throws IgniteCheckedException If storing value failed.
@@ -366,7 +368,9 @@ public interface GridCacheEntryEx {
MvccSnapshot mvccVer,
GridCacheOperation op,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException, GridCacheEntryRemovedException;
+ boolean noCreate,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
* @param tx Cache transaction.
@@ -374,6 +378,8 @@ public interface GridCacheEntryEx {
* @param topVer Topology version.
* @param mvccVer Mvcc version.
* @param needHistory Whether to collect rows created or affected by the current tx.
+ * @param filter Filter.
+ * @param retVal Previous value return flag.
* @return Tuple containing success flag and old value. If success is {@code false},
* then value is {@code null}.
* @throws IgniteCheckedException If storing value failed.
@@ -384,7 +390,9 @@ public interface GridCacheEntryEx {
UUID affNodeId,
AffinityTopologyVersion topVer,
MvccSnapshot mvccVer,
- boolean needHistory) throws IgniteCheckedException, GridCacheEntryRemovedException;
+ boolean needHistory,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException, GridCacheEntryRemovedException;
/**
* @param tx Transaction adapter.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
index 8fe559d..f58a3dc 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMapEntry.java
@@ -67,6 +67,7 @@ import org.apache.ignite.internal.processors.cache.transactions.IgniteTxEntry;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxKey;
import org.apache.ignite.internal.processors.cache.transactions.IgniteTxLocalAdapter;
import org.apache.ignite.internal.processors.cache.transactions.TxCounters;
+import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccUpdateDataRow;
import org.apache.ignite.internal.processors.cache.tree.mvcc.data.MvccUpdateResult;
import org.apache.ignite.internal.processors.cache.tree.mvcc.data.ResultType;
import org.apache.ignite.internal.processors.cache.version.GridCacheLazyPlainVersionedEntry;
@@ -1046,7 +1047,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
MvccSnapshot mvccVer,
GridCacheOperation op,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException, GridCacheEntryRemovedException {
+ boolean noCreate,
+ CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException, GridCacheEntryRemovedException {
assert tx != null;
final boolean valid = valid(tx.topologyVersion());
@@ -1087,7 +1090,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
assert val != null;
res = cctx.offheap().mvccUpdate(
- this, val, newVer, expireTime, mvccVer, tx.local(), needHistory, noCreate);
+ this, val, newVer, expireTime, mvccVer, tx.local(), needHistory, noCreate, filter, retVal);
assert res != null;
@@ -1100,7 +1103,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (res.resultType() == ResultType.VERSION_MISMATCH)
throw new IgniteSQLException("Mvcc version mismatch.", CONCURRENT_UPDATE);
- else if (noCreate && res.resultType() == ResultType.PREV_NULL)
+ else if (res.resultType() == ResultType.FILTERED || (noCreate && res.resultType() == ResultType.PREV_NULL))
return new GridCacheUpdateTxResult(false);
else if (res.resultType() == ResultType.LOCKED) {
unlockEntry();
@@ -1112,7 +1115,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
IgniteInternalFuture<?> lockFut = cctx.kernalContext().coordinators().waitFor(cctx, lockVer);
lockFut.listen(new MvccUpdateLockListener(tx, this, affNodeId, topVer, val, ttl0, mvccVer,
- op, needHistory, noCreate, resFut));
+ op, needHistory, noCreate, filter, retVal, resFut));
return new GridCacheUpdateTxResult(false, resFut);
}
@@ -1141,17 +1144,18 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
counters.incrementUpdateCounter(cctx.cacheId(), partition());
}
- if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
+ if (cctx.group().persistenceEnabled() && cctx.group().walEnabled()) {
logPtr = cctx.shared().wal().log(new DataRecord(new DataEntry(
- cctx.cacheId(),
- key,
- val,
- res.resultType() == ResultType.PREV_NULL ? CREATE : UPDATE,
- tx.nearXidVersion(),
- newVer,
- expireTime,
- key.partition(),
- 0L)));
+ cctx.cacheId(),
+ key,
+ val,
+ res.resultType() == ResultType.PREV_NULL ? CREATE : UPDATE,
+ tx.nearXidVersion(),
+ newVer,
+ expireTime,
+ key.partition(),
+ 0L)));
+ }
update(val, expireTime, ttl, newVer, true);
@@ -1172,6 +1176,14 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
GridCacheUpdateTxResult updRes = valid ? new GridCacheUpdateTxResult(true, 0L, logPtr) :
new GridCacheUpdateTxResult(false, logPtr);
+ CacheDataRow oldRow = ((MvccUpdateDataRow)res).oldRow();
+
+ if(retVal && (res.resultType() == ResultType.PREV_NOT_NULL || res.resultType() == ResultType.VERSION_FOUND)) {
+ assert oldRow != null;
+
+ updRes.prevValue(oldRow.value());
+ }
+
updRes.mvccHistory(res.history());
return updRes;
@@ -1183,7 +1195,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
UUID affNodeId,
AffinityTopologyVersion topVer,
MvccSnapshot mvccVer,
- boolean needHistory) throws IgniteCheckedException, GridCacheEntryRemovedException {
+ boolean needHistory,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException, GridCacheEntryRemovedException {
assert tx != null;
assert mvccVer != null;
@@ -1204,13 +1218,13 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
assert newVer != null : "Failed to get write version for tx: " + tx;
- res = cctx.offheap().mvccRemove(this, mvccVer, tx.local(), needHistory);
+ res = cctx.offheap().mvccRemove(this, mvccVer, tx.local(), needHistory, filter, retVal);
assert res != null;
if (res.resultType() == ResultType.VERSION_MISMATCH)
throw new IgniteSQLException("Mvcc version mismatch.", CONCURRENT_UPDATE);
- else if (res.resultType() == ResultType.PREV_NULL)
+ else if (res.resultType() == ResultType.PREV_NULL || res.resultType() == ResultType.FILTERED)
return new GridCacheUpdateTxResult(false);
else if (res.resultType() == ResultType.LOCKED) {
unlockEntry();
@@ -1222,7 +1236,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
IgniteInternalFuture<?> lockFut = cctx.kernalContext().coordinators().waitFor(cctx, lockVer);
lockFut.listen(new MvccRemoveLockListener(tx, this, affNodeId, topVer, mvccVer, needHistory,
- resFut));
+ resFut, retVal, filter));
return new GridCacheUpdateTxResult(false, resFut);
}
@@ -1265,6 +1279,14 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
GridCacheUpdateTxResult updRes = valid ? new GridCacheUpdateTxResult(true, 0L, logPtr) :
new GridCacheUpdateTxResult(false, logPtr);
+ CacheDataRow oldRow = ((MvccUpdateDataRow)res).oldRow();
+
+ if(retVal && (res.resultType() == ResultType.PREV_NOT_NULL || res.resultType() == ResultType.VERSION_FOUND)) {
+ assert oldRow != null;
+
+ updRes.prevValue(oldRow.value());
+ }
+
updRes.mvccHistory(res.history());
return updRes;
@@ -2264,12 +2286,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (updateMetrics &&
updateRes.outcome().updateReadMetrics() &&
needVal)
- cctx.cache().metrics0().onRead(oldVal != null);
+ cctx.cache().metrics0().onRead(oldVal != null);
if (updateMetrics && INVOKE_NO_OP.equals(updateRes.outcome()) && (transformOp || updateRes.transformed()))
cctx.cache().metrics0().onReadOnlyInvoke(oldVal != null);
else if (updateMetrics && REMOVE_NO_VAL.equals(updateRes.outcome())
- && (transformOp || updateRes.transformed()))
+ && (transformOp || updateRes.transformed()))
cctx.cache().metrics0().onInvokeRemove(oldVal != null);
switch (updateRes.outcome()) {
@@ -3521,11 +3543,11 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
// Detach value before index update.
val = cctx.kernalContext().cacheObjects().prepareForCache(val, cctx);
- if (val != null) {
- if (cctx.mvccEnabled())
- cctx.offheap().mvccInitialValue(this, val, newVer, expTime);
- else
- storeValue(val, expTime, newVer);
+ if (val != null) {
+ if (cctx.mvccEnabled())
+ cctx.offheap().mvccInitialValue(this, val, newVer, expTime);
+ else
+ storeValue(val, expTime, newVer);
if (deletedUnlocked())
deletedUnlocked(false);
@@ -4157,12 +4179,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
/**
- * Stores value inoffheap.*
+ * Stores value in off-heap.
+ *
* @param val Value.
* @param expireTime Expire time.
* @param ver New entry version.
* @param predicate Optional predicate.
- *
* @return {@code True} if storage was modified.
* @throws IgniteCheckedException If update failed.
*/
@@ -4299,7 +4321,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (detached())
return rawGet();
- for (;;) {
+ for (; ; ) {
GridCacheEntryEx e = cctx.cache().peekEx(key);
if (e == null)
@@ -4806,7 +4828,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (transformed)
cctx.cache().metrics0().onInvokeRemove(hasOldVal);
- } else if (op == READ && transformed)
+ }
+ else if (op == READ && transformed)
cctx.cache().metrics0().onReadOnlyInvoke(hasOldVal);
else {
cctx.cache().metrics0().onWrite();
@@ -4940,6 +4963,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
/** */
private final GridFutureAdapter<GridCacheUpdateTxResult> resFut;
+ /** Need previous value flag. */
+ private final boolean needVal;
+
+ /** Filter. */
+ private final CacheEntryPredicate filter;
+
/** */
private GridCacheMapEntry entry;
@@ -4950,7 +4979,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
AffinityTopologyVersion topVer,
MvccSnapshot mvccVer,
boolean needHistory,
- GridFutureAdapter<GridCacheUpdateTxResult> resFut) {
+ GridFutureAdapter<GridCacheUpdateTxResult> resFut,
+ boolean retVal,
+ @Nullable CacheEntryPredicate filter) {
this.tx = tx;
this.entry = entry;
this.topVer = topVer;
@@ -4958,6 +4989,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
this.mvccVer = mvccVer;
this.needHistory = needHistory;
this.resFut = resFut;
+ this.needVal = retVal;
+ this.filter = filter;
}
/** {@inheritDoc} */
@@ -4989,8 +5022,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
cctx.shared().database().checkpointReadLock();
try {
- res = cctx.offheap().mvccRemove(entry, mvccVer, tx.local(), needHistory);
- } finally {
+ res = cctx.offheap().mvccRemove(entry, mvccVer, tx.local(), needHistory, filter, needVal);
+ }
+ finally {
cctx.shared().database().checkpointReadUnlock();
}
@@ -5001,7 +5035,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
return;
}
- else if (res.resultType() == ResultType.PREV_NULL) {
+ else if (res.resultType() == ResultType.PREV_NULL || res.resultType() == ResultType.FILTERED) {
resFut.onDone(new GridCacheUpdateTxResult(false));
return;
@@ -5034,15 +5068,15 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
logPtr = cctx.shared().wal().log(new DataRecord(new DataEntry(
- cctx.cacheId(),
- entry.key(),
- null,
- DELETE,
- tx.nearXidVersion(),
- tx.writeVersion(),
- 0,
- entry.key().partition(),
- 0)));
+ cctx.cacheId(),
+ entry.key(),
+ null,
+ DELETE,
+ tx.nearXidVersion(),
+ tx.writeVersion(),
+ 0,
+ entry.key().partition(),
+ 0)));
entry.update(null, 0, 0, newVer, true);
@@ -5209,6 +5243,12 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
/** */
private final boolean noCreate;
+ /** Filter. */
+ private final CacheEntryPredicate filter;
+
+ /** Need previous value flag.*/
+ private final boolean needVal;
+
/** */
MvccUpdateLockListener(IgniteInternalTx tx,
GridCacheMapEntry entry,
@@ -5220,6 +5260,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
GridCacheOperation op,
boolean needHistory,
boolean noCreate,
+ CacheEntryPredicate filter,
+ boolean needVal,
GridFutureAdapter<GridCacheUpdateTxResult> resFut) {
this.tx = tx;
this.entry = entry;
@@ -5231,6 +5273,8 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
this.op = op;
this.needHistory = needHistory;
this.noCreate = noCreate;
+ this.filter = filter;
+ this.needVal = needVal;
this.resFut = resFut;
}
@@ -5279,8 +5323,9 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
try {
res = cctx.offheap().mvccUpdate(
- entry, val, newVer, expireTime, mvccVer, tx.local(), needHistory, noCreate);
- } finally {
+ entry, val, newVer, expireTime, mvccVer, tx.local(), needHistory, noCreate, filter, needVal);
+ }
+ finally {
cctx.shared().database().checkpointReadUnlock();
}
@@ -5329,15 +5374,15 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
if (cctx.group().persistenceEnabled() && cctx.group().walEnabled())
logPtr = cctx.shared().wal().log(new DataRecord(new DataEntry(
- cctx.cacheId(),
- entry.key(),
- val,
- res.resultType() == ResultType.PREV_NULL ? CREATE : UPDATE,
- tx.nearXidVersion(),
- newVer,
- expireTime,
- entry.key().partition(),
- 0L)));
+ cctx.cacheId(),
+ entry.key(),
+ val,
+ res.resultType() == ResultType.PREV_NULL ? CREATE : UPDATE,
+ tx.nearXidVersion(),
+ newVer,
+ expireTime,
+ entry.key().partition(),
+ 0L)));
entry.update(val, expireTime, ttl, newVer, true);
@@ -6007,8 +6052,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
@Nullable IgniteBiTuple<Object, Exception> invokeRes,
boolean readFromStore,
boolean transformed)
- throws IgniteCheckedException
- {
+ throws IgniteCheckedException {
GridCacheContext cctx = entry.context();
final CacheObject oldVal = entry.val;
@@ -6059,7 +6103,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
}
else {
- newSysTtl = newTtl = conflictCtx.ttl();
+ newSysTtl = newTtl = conflictCtx.ttl();
newSysExpireTime = newExpireTime = conflictCtx.expireTime();
}
@@ -6166,8 +6210,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
@Nullable IgniteBiTuple<Object, Exception> invokeRes,
boolean readFromStore,
boolean transformed)
- throws IgniteCheckedException
- {
+ throws IgniteCheckedException {
GridCacheContext cctx = entry.context();
CacheObject oldVal = entry.val;
@@ -6264,8 +6307,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
private GridCacheVersionConflictContext<?, ?> resolveConflict(
CacheObject newVal,
@Nullable IgniteBiTuple<Object, Exception> invokeRes)
- throws IgniteCheckedException
- {
+ throws IgniteCheckedException {
GridCacheContext cctx = entry.context();
// Cache is conflict-enabled.
@@ -6437,7 +6479,7 @@ public abstract class GridCacheMapEntry extends GridMetadataAwareAdapter impleme
}
catch (Exception e) {
if (e instanceof UnregisteredClassException || e instanceof UnregisteredBinaryTypeException)
- throw (IgniteException) e;
+ throw (IgniteException)e;
writeObj = invokeEntry.valObj;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
index b646cf9..4543dfd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheUpdateTxResult.java
@@ -40,14 +40,17 @@ public class GridCacheUpdateTxResult {
private GridLongList mvccWaitTxs;
/** */
- private GridFutureAdapter<GridCacheUpdateTxResult> fut;
+ private GridFutureAdapter<GridCacheUpdateTxResult> fut;
/** */
private WALPointer logPtr;
- /** */
+ /** Mvcc history. */
private List<MvccLinkAwareSearchRow> mvccHistory;
+ /** Previous value. */
+ private CacheObject prevVal;
+
/**
* Constructor.
*
@@ -158,6 +161,22 @@ public class GridCacheUpdateTxResult {
this.mvccHistory = mvccHistory;
}
+ /**
+ *
+ * @return Previous value.
+ */
+ @Nullable public CacheObject prevValue() {
+ return prevVal;
+ }
+
+ /**
+ *
+ * @param prevVal Previous value.
+ */
+ public void prevValue( @Nullable CacheObject prevVal) {
+ this.prevVal = prevVal;
+ }
+
/** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridCacheUpdateTxResult.class, this);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
index b4b6c9b..f576cc5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManager.java
@@ -276,6 +276,8 @@ public interface IgniteCacheOffheapManager {
* @param primary {@code True} if on primary node.
* @param needHistory Flag to collect history.
* @param noCreate Flag indicating that row should not be created if absent.
+ * @param filter Filter.
+ * @param retVal Flag to return previous value.
* @return Update result.
* @throws IgniteCheckedException If failed.
*/
@@ -287,13 +289,17 @@ public interface IgniteCacheOffheapManager {
MvccSnapshot mvccSnapshot,
boolean primary,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException;
+ boolean noCreate,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException;
/**
* @param entry Entry.
* @param mvccSnapshot MVCC snapshot.
* @param primary {@code True} if on primary node.
* @param needHistory Flag to collect history.
+ * @param filter Filter.
+ * @param retVal Flag to return previous value.
* @return Update result.
* @throws IgniteCheckedException If failed.
*/
@@ -301,7 +307,9 @@ public interface IgniteCacheOffheapManager {
GridCacheMapEntry entry,
MvccSnapshot mvccSnapshot,
boolean primary,
- boolean needHistory) throws IgniteCheckedException;
+ boolean needHistory,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException;
/**
* @param entry Entry.
@@ -788,9 +796,11 @@ public interface IgniteCacheOffheapManager {
* @param ver Version.
* @param expireTime Expire time.
* @param mvccSnapshot MVCC snapshot.
+ * @param filter Filter.
* @param primary {@code True} if update is executed on primary node.
* @param needHistory Flag to collect history.
* @param noCreate Flag indicating that row should not be created if absent.
+ * @param retVal Flag to return previous value.
* @return Update result.
* @throws IgniteCheckedException If failed.
*/
@@ -801,16 +811,20 @@ public interface IgniteCacheOffheapManager {
GridCacheVersion ver,
long expireTime,
MvccSnapshot mvccSnapshot,
+ @Nullable CacheEntryPredicate filter,
boolean primary,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException;
+ boolean noCreate,
+ boolean retVal) throws IgniteCheckedException;
/**
* @param cctx Cache context.
* @param key Key.
* @param mvccSnapshot MVCC snapshot.
+ * @param filter Filter.
* @param primary {@code True} if update is executed on primary node.
* @param needHistory Flag to collect history.
+ * @param retVal Flag to return previous value.
* @return List of transactions to wait for.
* @throws IgniteCheckedException If failed.
*/
@@ -818,8 +832,10 @@ public interface IgniteCacheOffheapManager {
GridCacheContext cctx,
KeyCacheObject key,
MvccSnapshot mvccSnapshot,
+ @Nullable CacheEntryPredicate filter,
boolean primary,
- boolean needHistory) throws IgniteCheckedException;
+ boolean needHistory,
+ boolean retVal) throws IgniteCheckedException;
/**
* @param cctx Cache context.
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
index 11e67d3..e0b9c06 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheOffheapManagerImpl.java
@@ -514,7 +514,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
MvccSnapshot mvccSnapshot,
boolean primary,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException {
+ boolean noCreate,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException {
if (entry.detached() || entry.isNear())
return null;
@@ -526,9 +528,11 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
ver,
expireTime,
mvccSnapshot,
+ filter,
primary,
needHistory,
- noCreate);
+ noCreate,
+ retVal);
}
/** {@inheritDoc} */
@@ -536,7 +540,9 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
GridCacheMapEntry entry,
MvccSnapshot mvccSnapshot,
boolean primary,
- boolean needHistory) throws IgniteCheckedException {
+ boolean needHistory,
+ @Nullable CacheEntryPredicate filter,
+ boolean retVal) throws IgniteCheckedException {
if (entry.detached() || entry.isNear())
return null;
@@ -545,8 +551,10 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
return dataStore(entry.localPartition()).mvccRemove(entry.context(),
entry.key(),
mvccSnapshot,
+ filter,
primary,
- needHistory);
+ needHistory,
+ retVal);
}
/** {@inheritDoc} */
@@ -1848,9 +1856,11 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
GridCacheVersion ver,
long expireTime,
MvccSnapshot mvccSnapshot,
+ @Nullable CacheEntryPredicate filter,
boolean primary,
boolean needHistory,
- boolean noCreate) throws IgniteCheckedException {
+ boolean noCreate,
+ boolean retVal) throws IgniteCheckedException {
assert mvccSnapshot != null;
assert primary || !needHistory;
@@ -1866,7 +1876,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
key.valueBytes(coCtx);
val.valueBytes(coCtx);
- MvccUpdateDataRow updateRow = new MvccUpdateDataRow(
+ MvccUpdateDataRow updateRow = new MvccUpdateDataRow(
cctx,
key,
val,
@@ -1875,11 +1885,13 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
expireTime,
mvccSnapshot,
null,
+ filter,
primary,
false,
needHistory,
// we follow fast update visit flow here if row cannot be created by current operation
- noCreate);
+ noCreate,
+ retVal);
assert cctx.shared().database().checkpointLockIsHeldByThread();
@@ -1890,17 +1902,15 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
if (res == ResultType.LOCKED // cannot update locked
|| res == ResultType.VERSION_MISMATCH) // cannot update on write conflict
return updateRow;
- else if (res == ResultType.VERSION_FOUND) {
+ else if (res == ResultType.VERSION_FOUND || // exceptional case
+ res == ResultType.FILTERED || // Operation should be skipped.
+ (res == ResultType.PREV_NULL && noCreate) // No op.
+ ) {
// Do nothing, except cleaning up not needed versions
cleanup(cctx, updateRow.cleanupRows());
return updateRow;
}
- else if (res == ResultType.PREV_NULL && noCreate) {
- cleanup(cctx, updateRow.cleanupRows());
-
- return updateRow;
- }
CacheDataRow oldRow = null;
@@ -1961,8 +1971,10 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
@Override public MvccUpdateResult mvccRemove(GridCacheContext cctx,
KeyCacheObject key,
MvccSnapshot mvccSnapshot,
+ @Nullable CacheEntryPredicate filter,
boolean primary,
- boolean needHistory) throws IgniteCheckedException {
+ boolean needHistory,
+ boolean retVal) throws IgniteCheckedException {
assert mvccSnapshot != null;
assert primary || mvccSnapshot.activeTransactions().size() == 0 : mvccSnapshot;
assert primary || !needHistory;
@@ -1987,10 +1999,12 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
0,
mvccSnapshot,
null,
+ filter,
primary,
false,
needHistory,
- true);
+ true,
+ retVal);
assert cctx.shared().database().checkpointLockIsHeldByThread();
@@ -2001,7 +2015,7 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
if (res == ResultType.LOCKED // cannot update locked
|| res == ResultType.VERSION_MISMATCH) // cannot update on write conflict
return updateRow;
- else if (res == ResultType.VERSION_FOUND) {
+ else if (res == ResultType.VERSION_FOUND || res == ResultType.FILTERED) {
// Do nothing, except cleaning up not needed versions
cleanup(cctx, updateRow.cleanupRows());
@@ -2051,9 +2065,11 @@ public class IgniteCacheOffheapManagerImpl implements IgniteCacheOffheapManager
0,
mvccSnapshot,
null,
+ null,
true,
true,
false,
+ false,
false);
assert cctx.shared().database().checkpointLockIsHeldByThread();
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
index 225fa81..4989efb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/IgniteCacheProxyImpl.java
@@ -348,7 +348,7 @@ public class IgniteCacheProxyImpl<K, V> extends AsyncSupportAdapter<IgniteCache<
/** {@inheritDoc} */
@Override public Lock lockAll(final Collection<? extends K> keys) {
- //TODO IGNITE-7764
+ //TODO: IGNITE-9324: add explicit locks support.
MvccUtils.verifyMvccOperationSupport(ctx, "Lock");
return new CacheLockImpl<>(ctx.gate(), delegate, ctx.operationContextPerCall(), keys);
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
index 4480dae..52638c0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java
@@ -60,6 +60,9 @@ import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLock
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTransactionalCache;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistFuture;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistRequest;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistResponse;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryEnlistFuture;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryEnlistRequest;
import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxQueryEnlistResponse;
@@ -188,7 +191,7 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxQueryEnlistResponse.class, new CI2<UUID, GridNearTxQueryEnlistResponse>() {
@Override public void apply(UUID nodeId, GridNearTxQueryEnlistResponse req) {
- processNearEnlistResponse(nodeId, req);
+ processNearTxQueryEnlistResponse(nodeId, req);
}
});
@@ -216,7 +219,21 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxQueryResultsEnlistResponse.class,
new CI2<UUID, GridNearTxQueryResultsEnlistResponse>() {
@Override public void apply(UUID nodeId, GridNearTxQueryResultsEnlistResponse req) {
- processNearTxEnlistResponse(nodeId, req);
+ processNearTxQueryResultsEnlistResponse(nodeId, req);
+ }
+ });
+
+ ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxEnlistRequest.class,
+ new CI2<UUID, GridNearTxEnlistRequest>() {
+ @Override public void apply(UUID nodeId, GridNearTxEnlistRequest req) {
+ processNearTxEnlistRequest(nodeId, req);
+ }
+ });
+
+ ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxEnlistResponse.class,
+ new CI2<UUID, GridNearTxEnlistResponse>() {
+ @Override public void apply(UUID nodeId, GridNearTxEnlistResponse msg) {
+ processNearTxEnlistResponse(nodeId, msg);
}
});
@@ -756,17 +773,6 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
/**
* @param nodeId Node ID.
- * @param res Response.
- */
- private void processNearEnlistResponse(UUID nodeId, final GridNearTxQueryEnlistResponse res) {
- GridNearTxQueryEnlistFuture fut = (GridNearTxQueryEnlistFuture)ctx.mvcc().versionedFuture(res.version(), res.futureId());
-
- if (fut != null)
- fut.onResult(nodeId, res);
- }
-
- /**
- * @param nodeId Node ID.
* @param req Request.
*/
private void processNearLockRequest(UUID nodeId, GridNearLockRequest req) {
@@ -801,7 +807,7 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
IgniteInternalFuture<?> f;
if (req.firstClientRequest()) {
- for (;;) {
+ for (; ; ) {
if (waitForExchangeFuture(nearNode, req))
return;
@@ -1079,9 +1085,9 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
if (top != null && needRemap(req.topologyVersion(), top.readyTopologyVersion())) {
if (log.isDebugEnabled()) {
log.debug("Client topology version mismatch, need remap lock request [" +
- "reqTopVer=" + req.topologyVersion() +
- ", locTopVer=" + top.readyTopologyVersion() +
- ", req=" + req + ']');
+ "reqTopVer=" + req.topologyVersion() +
+ ", locTopVer=" + top.readyTopologyVersion() +
+ ", req=" + req + ']');
}
GridNearLockResponse res = sendClientLockRemapResponse(nearNode,
@@ -1124,7 +1130,7 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
if (tx == null || !tx.init()) {
String msg = "Failed to acquire lock (transaction has been completed): " +
- req.version();
+ req.version();
U.warn(log, msg);
@@ -1401,7 +1407,7 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
int i = 0;
- for (ListIterator<GridCacheEntryEx> it = entries.listIterator(); it.hasNext();) {
+ for (ListIterator<GridCacheEntryEx> it = entries.listIterator(); it.hasNext(); ) {
GridCacheEntryEx e = it.next();
assert e != null;
@@ -1995,6 +2001,71 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
}
/**
+ * @param nodeId Node ID.
+ * @param req Request.
+ */
+ private void processNearTxEnlistRequest(UUID nodeId, final GridNearTxEnlistRequest req) {
+ assert nodeId != null;
+ assert req != null;
+
+ ClusterNode nearNode = ctx.discovery().node(nodeId);
+
+ GridDhtTxLocal tx;
+
+ try {
+ tx = initTxTopologyVersion(nodeId,
+ nearNode,
+ req.version(),
+ req.futureId(),
+ req.miniId(),
+ req.firstClientRequest(),
+ req.topologyVersion(),
+ req.threadId(),
+ req.txTimeout(),
+ req.subjectId(),
+ req.taskNameHash());
+ }
+ catch (IgniteCheckedException | IgniteException ex) {
+ GridNearTxEnlistResponse res = new GridNearTxEnlistResponse(req.cacheId(),
+ req.futureId(),
+ req.miniId(),
+ req.version(),
+ ex);
+
+ try {
+ ctx.io().send(nearNode, res, ctx.ioPolicy());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(log, "Failed to send near enlist response [" +
+ "txId=" + req.version() +
+ ", node=" + nodeId +
+ ", res=" + res + ']', e);
+ }
+
+ return;
+ }
+
+ GridDhtTxEnlistFuture fut = new GridDhtTxEnlistFuture(
+ nodeId,
+ req.version(),
+ req.mvccSnapshot(),
+ req.threadId(),
+ req.futureId(),
+ req.miniId(),
+ tx,
+ req.timeout(),
+ ctx,
+ req.rows(),
+ req.operation(),
+ req.filter(),
+ req.needRes());
+
+ fut.listen(NearTxResultHandler.instance());
+
+ fut.init();
+ }
+
+ /**
* @param nodeId Near node id.
* @param nearNode Near node.
* @param nearLockVer Near lock version.
@@ -2125,7 +2196,30 @@ public abstract class GridDhtTransactionalCacheAdapter<K, V> extends GridDhtCach
* @param nodeId Node ID.
* @param res Response.
*/
- private void processNearTxEnlistResponse(UUID nodeId, final GridNearTxQueryResultsEnlistResponse res) {
+ private void processNearTxEnlistResponse(UUID nodeId, final GridNearTxEnlistResponse res) {
+ GridNearTxEnlistFuture fut = (GridNearTxEnlistFuture)
+ ctx.mvcc().versionedFuture(res.version(), res.futureId());
+
+ if (fut != null)
+ fut.onResult(nodeId, res);
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param res Response.
+ */
+ private void processNearTxQueryEnlistResponse(UUID nodeId, final GridNearTxQueryEnlistResponse res) {
+ GridNearTxQueryEnlistFuture fut = (GridNearTxQueryEnlistFuture)ctx.mvcc().versionedFuture(res.version(), res.futureId());
+
+ if (fut != null)
+ fut.onResult(nodeId, res);
+ }
+
+ /**
+ * @param nodeId Node ID.
+ * @param res Response.
+ */
+ private void processNearTxQueryResultsEnlistResponse(UUID nodeId, final GridNearTxQueryResultsEnlistResponse res) {
GridNearTxQueryResultsEnlistFuture fut = (GridNearTxQueryResultsEnlistFuture)
ctx.mvcc().versionedFuture(res.version(), res.futureId());
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
index ad164e7..64f966d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxAbstractEnlistFuture.java
@@ -37,6 +37,7 @@ import org.apache.ignite.internal.cluster.ClusterTopologyCheckedException;
import org.apache.ignite.internal.pagemem.wal.WALPointer;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheEntryInfoCollection;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
import org.apache.ignite.internal.processors.cache.CacheObject;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
import org.apache.ignite.internal.processors.cache.GridCacheEntryInfo;
@@ -77,11 +78,10 @@ import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
/**
- * Abstract future processing transaction enlisting and locking
- * of entries produced with DML and SELECT FOR UPDATE queries.
+ * Abstract future processing transaction enlisting and locking.
*/
-public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapter<Long>
- implements DhtLockFuture<Long> {
+public abstract class GridDhtTxAbstractEnlistFuture<T> extends GridCacheFutureAdapter<T>
+ implements DhtLockFuture<T> {
/** Done field updater. */
private static final AtomicIntegerFieldUpdater<GridDhtTxAbstractEnlistFuture> DONE_UPD =
AtomicIntegerFieldUpdater.newUpdater(GridDhtTxAbstractEnlistFuture.class, "done");
@@ -134,9 +134,6 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
/** */
protected final MvccSnapshot mvccSnapshot;
- /** Processed entries count. */
- protected long cnt;
-
/** New DHT nodes. */
protected Set<UUID> newDhtNodes = Collections.newSetFromMap(new ConcurrentHashMap<>());
@@ -146,6 +143,9 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
/** Near lock version. */
protected final GridCacheVersion nearLockVer;
+ /** Filter. */
+ private final CacheEntryPredicate filter;
+
/** Timeout object. */
@GridToStringExclude
protected LockTimeoutObject timeoutObj;
@@ -202,6 +202,7 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
* @param tx Transaction.
* @param timeout Lock acquisition timeout.
* @param cctx Cache context.
+ * @param filter Filter.
*/
protected GridDhtTxAbstractEnlistFuture(UUID nearNodeId,
GridCacheVersion nearLockVer,
@@ -212,7 +213,8 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
@Nullable int[] parts,
GridDhtTxLocalAdapter tx,
long timeout,
- GridCacheContext<?, ?> cctx) {
+ GridCacheContext<?, ?> cctx,
+ @Nullable CacheEntryPredicate filter) {
assert tx != null;
assert timeout >= 0;
assert nearNodeId != null;
@@ -229,6 +231,7 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
this.timeout = timeout;
this.tx = tx;
this.parts = parts;
+ this.filter = filter;
lockVer = tx.xidVersion();
@@ -238,12 +241,38 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
}
/**
+ * Gets source to be updated iterator.
+ *
* @return iterator.
* @throws IgniteCheckedException If failed.
*/
protected abstract UpdateSourceIterator<?> createIterator() throws IgniteCheckedException;
/**
+ * Gets query result.
+ *
+ * @return Query result.
+ */
+ protected abstract T result0();
+
+ /**
+ * Gets need previous value flag.
+ *
+ * @return {@code True} if previous value is required.
+ */
+ public boolean needResult() {
+ return false;
+ }
+
+ /**
+ * Entry processed callback.
+ *
+ * @param key Entry key.
+ * @param res Update result.
+ */
+ protected abstract void onEntryProcessed(KeyCacheObject key, GridCacheUpdateTxResult res);
+
+ /**
*
*/
public void init() {
@@ -291,14 +320,14 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
boolean added = cctx.mvcc().addFuture(this, futId);
- assert added;
-
if (isDone()) {
cctx.mvcc().removeFuture(futId);
return;
}
+ assert added;
+
if (timeoutObj != null)
cctx.time().addTimeoutObject(timeoutObj);
@@ -310,12 +339,15 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
if (!it.hasNext()) {
U.close(it, log);
- onDone(0L);
+ onDone(result0());
return;
}
- tx.addActiveCache(cctx, false);
+ if(!tx.implicitSingle())
+ tx.addActiveCache(cctx, false);
+ else // Nothing to do for single update.
+ assert tx.txState().cacheIds().contains(cctx.cacheId()) && tx.txState().cacheIds().size() == 1;
this.it = it;
}
@@ -391,7 +423,9 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
cctx.localNodeId(),
topVer,
mvccSnapshot,
- isMoving(key.partition()));
+ isMoving(key.partition()),
+ filter,
+ needResult());
break;
@@ -407,7 +441,9 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
mvccSnapshot,
op.cacheOperation(),
isMoving(key.partition()),
- op.noCreate());
+ op.noCreate(),
+ filter,
+ needResult());
break;
@@ -493,7 +529,7 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
}
if (noPendingRequests()) {
- onDone(cnt);
+ onDone(result0());
return;
}
@@ -569,11 +605,11 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
if (ptr0 != null)
walPtr = ptr0;
+ onEntryProcessed(entry.key(), updRes);
+
if (!updRes.success())
return;
- cnt++;
-
if (op != EnlistOperation.LOCK)
addToBatch(entry.key(), val, updRes.mvccHistory(), entry.context().cacheId());
}
@@ -980,7 +1016,7 @@ public abstract class GridDhtTxAbstractEnlistFuture extends GridCacheFutureAdapt
}
/** {@inheritDoc} */
- @Override public boolean onDone(@Nullable Long res, @Nullable Throwable err) {
+ @Override public boolean onDone(@Nullable T res, @Nullable Throwable err) {
assert res != null || err != null;
if (!DONE_UPD.compareAndSet(this, 0, 1))
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java
new file mode 100644
index 0000000..58d6b15
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxEnlistFuture.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.processors.cache.CacheEntryPredicate;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.processors.query.EnlistOperation;
+import org.apache.ignite.internal.processors.query.UpdateSourceIterator;
+import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Future processing transaction enlisting and locking of entries produces by cache API operations.
+ */
+public final class GridDhtTxEnlistFuture extends GridDhtTxAbstractEnlistFuture<GridCacheReturn> implements UpdateSourceIterator<Object> {
+ /** Enlist operation. */
+ private EnlistOperation op;
+
+ /** Source iterator. */
+ private Iterator<Object> it;
+
+ /** Future result. */
+ private GridCacheReturn res;
+
+ /** Need result flag. If {@code True} previous value should be returned as well. */
+ private boolean needRes;
+
+ /**
+ * Constructor.
+ *
+ * @param nearNodeId Near node ID.
+ * @param nearLockVer Near lock version.
+ * @param mvccSnapshot Mvcc snapshot.
+ * @param threadId Thread ID.
+ * @param nearFutId Near future id.
+ * @param nearMiniId Near mini future id.
+ * @param tx Transaction.
+ * @param timeout Lock acquisition timeout.
+ * @param cctx Cache context.
+ * @param rows Collection of rows.
+ * @param op Operation.
+ * @param filter Filter.
+ * @param needRes Return previous value flag.
+ */
+ public GridDhtTxEnlistFuture(UUID nearNodeId,
+ GridCacheVersion nearLockVer,
+ MvccSnapshot mvccSnapshot,
+ long threadId,
+ IgniteUuid nearFutId,
+ int nearMiniId,
+ GridDhtTxLocalAdapter tx,
+ long timeout,
+ GridCacheContext<?, ?> cctx,
+ Collection<Object> rows,
+ EnlistOperation op,
+ @Nullable CacheEntryPredicate filter,
+ boolean needRes) {
+ super(nearNodeId,
+ nearLockVer,
+ mvccSnapshot,
+ threadId,
+ nearFutId,
+ nearMiniId,
+ null,
+ tx,
+ timeout,
+ cctx,
+ filter);
+
+ this.op = op;
+ this.needRes = needRes;
+
+ it = rows.iterator();
+
+ res = new GridCacheReturn(cctx.localNodeId().equals(nearNodeId), false);
+
+ skipNearNodeUpdates = true;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected UpdateSourceIterator<?> createIterator() throws IgniteCheckedException {
+ return this;
+ }
+
+ /** {@inheritDoc} */
+ @Override @Nullable protected GridCacheReturn result0() {
+ return res;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onEntryProcessed(KeyCacheObject key, GridCacheUpdateTxResult txRes) {
+ if (needRes && txRes.success())
+ res.set(cctx, txRes.prevValue(), txRes.success(), true);
+ else
+ res.success(txRes.success());
+ }
+
+ /** {@inheritDoc} */
+ public boolean needResult() {
+ return needRes;
+ }
+
+ /** {@inheritDoc} */
+ @Override public EnlistOperation operation() {
+ return op;
+ }
+
+ /** {@inheritDoc} */
+ public boolean hasNextX() {
+ return it.hasNext();
+ }
+
+ /** {@inheritDoc} */
+ public Object nextX() {
+ return it.next();
+ }
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GridDhtTxEnlistFuture.class, this);
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryAbstractEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryAbstractEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryAbstractEnlistFuture.java
new file mode 100644
index 0000000..0a26d75
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryAbstractEnlistFuture.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.UUID;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheUpdateTxResult;
+import org.apache.ignite.internal.processors.cache.KeyCacheObject;
+import org.apache.ignite.internal.processors.cache.mvcc.MvccSnapshot;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.lang.IgniteUuid;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * Abstract future processing transaction enlisting and locking of entries produced with DML and SELECT FOR UPDATE
+ * queries.
+ */
+public abstract class GridDhtTxQueryAbstractEnlistFuture extends GridDhtTxAbstractEnlistFuture<Long> {
+ /** Processed entries count. */
+ protected long cnt;
+
+ /**
+ * Constructor.
+ *
+ * @param nearNodeId Near node ID.
+ * @param nearLockVer Near lock version.
+ * @param mvccSnapshot Mvcc snapshot.
+ * @param threadId Thread ID.
+ * @param nearFutId Near future id.
+ * @param nearMiniId Near mini future id.
+ * @param parts Partitions.
+ * @param tx Transaction.
+ * @param timeout Lock acquisition timeout.
+ * @param cctx Cache context.
+ */
+ protected GridDhtTxQueryAbstractEnlistFuture(UUID nearNodeId,
+ GridCacheVersion nearLockVer,
+ MvccSnapshot mvccSnapshot,
+ long threadId,
+ IgniteUuid nearFutId,
+ int nearMiniId,
+ @Nullable int[] parts,
+ GridDhtTxLocalAdapter tx,
+ long timeout,
+ GridCacheContext<?, ?> cctx) {
+ super(nearNodeId,
+ nearLockVer,
+ mvccSnapshot,
+ threadId,
+ nearFutId,
+ nearMiniId,
+ null,
+ tx,
+ timeout,
+ cctx, null);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected Long result0() {
+ return cnt;
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void onEntryProcessed(KeyCacheObject key, GridCacheUpdateTxResult res) {
+ if(res.success())
+ cnt++;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistFuture.java
index dd30855..ed792f0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryEnlistFuture.java
@@ -17,7 +17,6 @@
package org.apache.ignite.internal.processors.cache.distributed.dht;
-import java.util.Objects;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -29,9 +28,9 @@ import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.lang.IgniteUuid;
/**
- * Cache lock future.
+ * Cache query lock future.
*/
-public final class GridDhtTxQueryEnlistFuture extends GridDhtTxAbstractEnlistFuture {
+public final class GridDhtTxQueryEnlistFuture extends GridDhtTxQueryAbstractEnlistFuture {
/** Involved cache ids. */
private final int[] cacheIds;
@@ -116,24 +115,6 @@ public final class GridDhtTxQueryEnlistFuture extends GridDhtTxAbstractEnlistFut
}
/** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridDhtTxQueryEnlistFuture future = (GridDhtTxQueryEnlistFuture)o;
-
- return Objects.equals(futId, future.futId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return futId.hashCode();
- }
-
- /** {@inheritDoc} */
@Override public String toString() {
return S.toString(GridDhtTxQueryEnlistFuture.class, this);
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryResultsEnlistFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryResultsEnlistFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryResultsEnlistFuture.java
index b3d15d4..c6140fb 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryResultsEnlistFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxQueryResultsEnlistFuture.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.processors.cache.distributed.dht;
import java.util.Collection;
import java.util.Iterator;
-import java.util.Objects;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.internal.processors.cache.GridCacheContext;
@@ -34,13 +33,11 @@ import org.apache.ignite.lang.IgniteUuid;
* Future processing transaction enlisting and locking of entries
* produces by complex DML queries with reduce step.
*/
-public final class GridDhtTxQueryResultsEnlistFuture extends GridDhtTxAbstractEnlistFuture implements UpdateSourceIterator<Object> {
- /** */
- private static final long serialVersionUID = -4933550335145438798L;
- /** */
+public final class GridDhtTxQueryResultsEnlistFuture extends GridDhtTxQueryAbstractEnlistFuture implements UpdateSourceIterator<Object> {
+ /** Enlist operation. */
private EnlistOperation op;
- /** */
+ /** Source iterator. */
private Iterator<Object> it;
/**
@@ -91,29 +88,6 @@ public final class GridDhtTxQueryResultsEnlistFuture extends GridDhtTxAbstractEn
}
/** {@inheritDoc} */
- @Override public boolean equals(Object o) {
- if (this == o)
- return true;
-
- if (o == null || getClass() != o.getClass())
- return false;
-
- GridDhtTxQueryResultsEnlistFuture future = (GridDhtTxQueryResultsEnlistFuture)o;
-
- return Objects.equals(futId, future.futId);
- }
-
- /** {@inheritDoc} */
- @Override public int hashCode() {
- return futId.hashCode();
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(GridDhtTxQueryResultsEnlistFuture.class, this);
- }
-
- /** {@inheritDoc} */
@Override public EnlistOperation operation() {
return op;
}
@@ -127,4 +101,9 @@ public final class GridDhtTxQueryResultsEnlistFuture extends GridDhtTxAbstractEn
@Override public Object nextX() {
return it.next();
}
+
+ /** {@inheritDoc} */
+ @Override public String toString() {
+ return S.toString(GridDhtTxQueryResultsEnlistFuture.class, this);
+ }
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
index 6662a1c..9883f6d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTxRemote.java
@@ -440,6 +440,8 @@ public class GridDhtTxRemote extends GridDistributedTxRemoteAdapter {
ctx.localNodeId(),
topologyVersion(),
snapshot,
+ false,
+ null,
false);
break;
@@ -456,6 +458,8 @@ public class GridDhtTxRemote extends GridDistributedTxRemoteAdapter {
snapshot,
op.cacheOperation(),
false,
+ false,
+ null,
false);
break;
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
index f5689f9..5d3bef2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridPartitionedSingleGetFuture.java
@@ -389,7 +389,8 @@ public class GridPartitionedSingleGetFuture extends GridCacheFutureAdapter<Objec
boolean skipEntry = readNoEntry;
if (readNoEntry) {
- CacheDataRow row = mvccSnapshot != null ? cctx.offheap().mvccRead(cctx, key, mvccSnapshot) :
+ CacheDataRow row = mvccSnapshot != null ?
+ cctx.offheap().mvccRead(cctx, key, mvccSnapshot) :
cctx.offheap().read(cctx, key);
if (row != null) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/f7f834bf/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/NearTxResultHandler.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/NearTxResultHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/NearTxResultHandler.java
new file mode 100644
index 0000000..0bc00e1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/NearTxResultHandler.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.cache.distributed.dht;
+
+import java.util.UUID;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.internal.IgniteInternalFuture;
+import org.apache.ignite.internal.processors.cache.GridCacheContext;
+import org.apache.ignite.internal.processors.cache.GridCacheReturn;
+import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxEnlistResponse;
+import org.apache.ignite.internal.processors.cache.version.GridCacheVersion;
+import org.apache.ignite.internal.util.lang.GridClosureException;
+import org.apache.ignite.internal.util.typedef.CI1;
+import org.apache.ignite.internal.util.typedef.internal.CU;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteUuid;
+
+/**
+ * Response factory.
+ */
+public final class NearTxResultHandler implements CI1<IgniteInternalFuture<GridCacheReturn>> {
+ /** */
+ private static final long serialVersionUID = 0;
+
+ /** Singleton instance.*/
+ private static final NearTxResultHandler INSTANCE = new NearTxResultHandler();
+
+ /** Constructor. */
+ private NearTxResultHandler() {
+ }
+
+ /**
+ * @return Handler instance.
+ */
+ public static NearTxResultHandler instance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Response factory method.
+ *
+ * @param future Enlist future.
+ * @return Enlist response.
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T createResponse(IgniteInternalFuture<?> future) {
+ assert future != null;
+
+ Class<?> clazz = future.getClass();
+
+ if (clazz == GridDhtTxEnlistFuture.class)
+ return (T)createResponse((GridDhtTxEnlistFuture)future);
+ else
+ throw new IllegalStateException();
+ }
+
+ /**
+ * Response factory method.
+ *
+ * @param fut Enlist future.
+ * @return Enlist response.
+ */
+ public static GridNearTxEnlistResponse createResponse(GridDhtTxEnlistFuture fut) {
+ try {
+ GridCacheReturn res = fut.get();
+
+ GridCacheVersion ver = null;
+ IgniteUuid id = null;
+
+ if (fut.hasNearNodeUpdates) {
+ ver = fut.cctx.tm().mappedVersion(fut.nearLockVer);
+
+ id = fut.futId;
+ }
+
+ return new GridNearTxEnlistResponse(fut.cctx.cacheId(), fut.nearFutId, fut.nearMiniId,
+ fut.nearLockVer, res, ver, id, fut.newDhtNodes);
+ }
+ catch (IgniteCheckedException e) {
+ return new GridNearTxEnlistResponse(fut.cctx.cacheId(), fut.nearFutId, fut.nearMiniId, fut.nearLockVer, e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void apply(IgniteInternalFuture<GridCacheReturn> fut0) {
+ GridDhtTxAbstractEnlistFuture fut = (GridDhtTxAbstractEnlistFuture)fut0;
+
+ GridCacheContext<?, ?> cctx = fut.cctx;
+ GridDhtTxLocal tx = (GridDhtTxLocal)fut.tx;
+ UUID nearNodeId = fut.nearNodeId;
+
+ GridNearTxEnlistResponse res = createResponse(fut);
+
+ try {
+ cctx.io().send(nearNodeId, res, cctx.ioPolicy());
+ }
+ catch (IgniteCheckedException e) {
+ U.error(fut.log, "Failed to send near enlist response (will rollback transaction) [" +
+ "tx=" + CU.txString(tx) +
+ ", node=" + nearNodeId +
+ ", res=" + res + ']', e);
+
+ try {
+ tx.rollbackDhtLocalAsync();
+ }
+ catch (Throwable e1) {
+ e.addSuppressed(e1);
+ }
+
+ throw new GridClosureException(e);
+ }
+ }
+}